public static ObjectObjectHashMap<String, CollectionStatistics> readFieldStats(StreamInput in, ObjectObjectHashMap<String, CollectionStatistics> fieldStatistics) throws IOException {
    final int numFieldStatistics = in.readVInt();
    if (fieldStatistics == null) {
        fieldStatistics = HppcMaps.newNoNullKeysMap(numFieldStatistics);
    }
    for (int i = 0; i < numFieldStatistics; i++) {
        final String field = in.readString();
        assert field != null;
        final long maxDoc = in.readVLong();
        final long docCount = subOne(in.readVLong());
        final long sumTotalTermFreq = subOne(in.readVLong());
        final long sumDocFreq = subOne(in.readVLong());
        CollectionStatistics stats = new CollectionStatistics(field, maxDoc, docCount, sumTotalTermFreq, sumDocFreq);
        fieldStatistics.put(field, stats);
    }
    return fieldStatistics;
}
 

@Override
public synchronized void copyContextFrom(HasContext other) {
    if (other == null) {
        return;
    }

    synchronized (other) {
        ImmutableOpenMap<Object, Object> otherContext = other.getContext();
        if (otherContext == null) {
            return;
        }
        if (context == null) {
            ObjectObjectHashMap<Object, Object> map = new ObjectObjectHashMap<>(other.getContext().size());
            map.putAll(otherContext);
            this.context = map;
        } else {
            context.putAll(otherContext);
        }
    }
}
 

@Override
public Collection<KVPair> transform(Collection<KVPair> buffer) throws Exception{
    if(LOG.isTraceEnabled())
        SpliceLogUtils.trace(LOG,"transform buffer rows=%d",buffer.size());
    Collection<KVPair> newList=new ArrayList<>(buffer.size());
    for(KVPair indexPair : buffer){
        for(Pair<WriteContext, ObjectObjectHashMap<KVPair, KVPair>> pair : sharedMainMutationList){
            KVPair base=pair.getSecond().get(indexPair);
            if(base!=null){
                if(pair.getFirst().canRun(base))
                    newList.add(indexPair);
                break;
            }
        }
    }
    if(LOG.isTraceEnabled())
        SpliceLogUtils.trace(LOG,"transform returns buffer rows=%d",newList.size());
    return newList;
}
 

public CallBuffer<KVPair> getWriteBuffer(byte[] conglomBytes,
                                         WriteContext context,
                                         ObjectObjectHashMap<KVPair, KVPair> indexToMainMutationMap,
                                         int maxSize,
                                         boolean useAsyncWriteBuffers,
                                         TxnView txn, byte[] token) throws Exception {

    CallBuffer<KVPair> writeBuffer = sharedCallBufferMap.get(conglomBytes);
    if (writeBuffer == null) {
        writeBuffer = createKvPairCallBuffer(conglomBytes, context, indexToMainMutationMap, maxSize, useAsyncWriteBuffers, txn, token);
    } else {
        ((SharedPreFlushHook) writeBuffer.getPreFlushHook()).registerContext(context, indexToMainMutationMap);
        writeBuffer.getWriteConfiguration().registerContext(context, indexToMainMutationMap);
    }
    return writeBuffer;
}
 

public static void writeFieldStats(StreamOutput out, ObjectObjectHashMap<String, CollectionStatistics> fieldStatistics) throws IOException {
    out.writeVInt(fieldStatistics.size());

    for (ObjectObjectCursor<String, CollectionStatistics> c : fieldStatistics) {
        out.writeString(c.key);
        CollectionStatistics statistics = c.value;
        assert statistics.maxDoc() >= 0;
        out.writeVLong(statistics.maxDoc());
        out.writeVLong(addOne(statistics.docCount()));
        out.writeVLong(addOne(statistics.sumTotalTermFreq()));
        out.writeVLong(addOne(statistics.sumDocFreq()));
    }
}
 

/** Add fields so that they can later be fetched using {@link #getByKey(Object)}. */
public void addWithKey(Object key, IndexableField field) {
    if (keyedFields == null) {
        keyedFields = new ObjectObjectHashMap<>();
    } else if (keyedFields.containsKey(key)) {
        throw new IllegalStateException("Only one field can be stored per key");
    }
    keyedFields.put(key, field);
    add(field);
}
 

/**
 * Wraps the given map and prevent adding of <code>null</code> keys.
 * 
 * @param expectedElements
 *          The expected number of elements guaranteed not to cause buffer
 *          expansion (inclusive).
 */
public static <K, V> ObjectObjectHashMap<K, V> ensureNoNullKeys(int expectedElements) {
    return new ObjectObjectHashMap<K, V>(expectedElements) {
        @Override
        public V put(K key, V value) {
            if (key == null) {
                throw new IllegalArgumentException("Map key must not be null");
            }
            return super.put(key, value);
        }
    };
}
 

@SuppressWarnings("unchecked")
@Override
public final synchronized <V> V putInContext(Object key, Object value) {
    if (context == null) {
        context = new ObjectObjectHashMap<>(2);
    }
    return (V) context.put(key, value);
}
 

@Override
public final synchronized void putAllInContext(ObjectObjectAssociativeContainer<Object, Object> map) {
    if (map == null) {
        return;
    }
    if (context == null) {
        context = new ObjectObjectHashMap<>(map);
    } else {
        context.putAll(map);
    }
}
 

protected void initMatrix(
    final ObjectObjectHashMap<Double, IntFloatHashMap> matrix,
    final Set<Double> uniqueLabels,
    final BagOfPattern[] bag) {
  for (Double label : uniqueLabels) {
    IntFloatHashMap stat = matrix.get(label);
    if (stat == null) {
      matrix.put(label, new IntFloatHashMap(bag[0].bag.size() * bag.length));
    } else {
      stat.clear();
    }
  }
}
 

@Override
public void setup(final int[] keys, final float fillFactor, final int oneFailureOutOf ) {
    super.setup( keys, fillFactor, oneFailureOutOf );
    m_map = new ObjectObjectHashMap<>( keys.length, fillFactor );
    for (Integer key : m_keys)
        m_map.put(new Integer( key % oneFailureOutOf == 0 ? key + 1 : key ), key);
}
 

@Override
public int test() {
    final ObjectObjectMap<Integer, Integer> m_map = new ObjectObjectHashMap<>( m_keys.length, m_fillFactor );
    for ( int i = 0; i < m_keys.length; ++i )
        m_map.put( m_keys[ i ], m_keys[ i ] );
    for ( int i = 0; i < m_keys2.length; ++i )
        m_map.put( m_keys2[ i ], m_keys2[ i ] );
    return m_map.size();
}
 

@Override
public int test() {
    final ObjectObjectMap<Integer, Integer> m_map = new ObjectObjectHashMap<>( m_keys.length / 2 + 1, m_fillFactor );
    int add = 0, remove = 0;
    while ( add < m_keys.length )
    {
        m_map.put( m_keys[ add ], m_keys[ add ] );
        ++add;
        m_map.put( m_keys[ add ], m_keys[ add ] );
        ++add;
        m_map.remove( m_keys2[ remove++ ] );
    }
    return m_map.size();
}
 

private CallBuffer<KVPair> createKvPairCallBuffer(byte[] conglomBytes,
                                                  WriteContext context,
                                                  ObjectObjectHashMap<KVPair, KVPair> indexToMainMutationMap,
                                                  int maxSize,
                                                  boolean useAsyncWriteBuffers,
                                                  TxnView txn, byte[] token) throws IOException{
    SharedPreFlushHook hook = new SharedPreFlushHook();
    WriteConfiguration writeConfiguration=writerPool.defaultWriteConfiguration();
    WriteConfiguration wc = new SharedWriteConfiguration(writeConfiguration.getMaximumRetries(),
            writeConfiguration.getPause(),
            writeConfiguration.getExceptionFactory());
    if (context.skipConflictDetection() || context.skipWAL()) {
        wc = new UnsafeWriteConfiguration(wc, context.skipConflictDetection(), context.skipWAL());
    }
    if (context.rollforward()) {
        wc = new RollforwardWriteConfiguration(wc);
    }
    hook.registerContext(context, indexToMainMutationMap);
    wc.registerContext(context, indexToMainMutationMap);
    CallBuffer<KVPair> writeBuffer;
    if (useAsyncWriteBuffers) {
        writeBuffer = writerPool.writeBuffer(partitionFactory.getTable(conglomBytes), txn,token, hook, wc, false);
    } else {
        writeBuffer = writerPool.synchronousWriteBuffer(partitionFactory.getTable(conglomBytes), txn, token, hook, wc, maxSize, false);
    }
    sharedCallBufferMap.put(conglomBytes, writeBuffer);
    return writeBuffer;
}
 

@Override
public CallBuffer<KVPair> getSharedWriteBuffer(byte[] conglomBytes,
                                               ObjectObjectHashMap<KVPair, KVPair> indexToMainMutationMap,
                                               int maxSize, boolean useAsyncWriteBuffers, TxnView txn, byte[] token) throws Exception {
    assert indexSharedCallBuffer != null;
    return indexSharedCallBuffer.getWriteBuffer(conglomBytes, this, indexToMainMutationMap, maxSize, useAsyncWriteBuffers, txn, token);
}
 

/** Add fields so that they can later be fetched using {@link #getByKey(Object)}. */
public void addWithKey(Object key, IndexableField field) {
    if (keyedFields == null) {
        keyedFields = new ObjectObjectHashMap<>();
    } else if (keyedFields.containsKey(key)) {
        throw new IllegalStateException("Only one field can be stored per key");
    }
    keyedFields.put(key, field);
    add(field);
}
 

public AggregatedDfs(ObjectObjectHashMap<Term, TermStatistics> termStatistics, ObjectObjectHashMap<String, CollectionStatistics> fieldStatistics, long maxDoc) {
    this.termStatistics = termStatistics;
    this.fieldStatistics = fieldStatistics;
    this.maxDoc = maxDoc;
}
 

public ObjectObjectHashMap<Term, TermStatistics> termStatistics() {
    return termStatistics;
}
 

public ObjectObjectHashMap<String, CollectionStatistics> fieldStatistics() {
    return fieldStatistics;
}
 

public ObjectObjectHashMap<String, CollectionStatistics> fieldStatistics() {
    return fieldStatistics;
}
 

public static ObjectObjectHashMap<String, CollectionStatistics> readFieldStats(StreamInput in) throws IOException {
    return readFieldStats(in, null);
}
 

/**
 * Returns a new map with a default initial capacity.
 */
public static <K, V> ObjectObjectHashMap<K, V> newMap() {
    return newMap(16);
}
 

/**
 * Returns a map like {@link #newMap()} that does not accept <code>null</code> keys
 */
public static <K, V> ObjectObjectHashMap<K, V> newNoNullKeysMap() {
    return ensureNoNullKeys(16);
}
 

public ClusterStatsIndices(ClusterStatsNodeResponse[] nodeResponses) {
    ObjectObjectHashMap<String, ShardStats> countsPerIndex = new ObjectObjectHashMap<>();

    this.docs = new DocsStats();
    this.store = new StoreStats();
    this.fieldData = new FieldDataStats();
    this.queryCache = new QueryCacheStats();
    this.completion = new CompletionStats();
    this.segments = new SegmentsStats();
    this.percolate = new PercolateStats();

    for (ClusterStatsNodeResponse r : nodeResponses) {
        for (org.elasticsearch.action.admin.indices.stats.ShardStats shardStats : r.shardsStats()) {
            ShardStats indexShardStats = countsPerIndex.get(shardStats.getShardRouting().getIndex());
            if (indexShardStats == null) {
                indexShardStats = new ShardStats();
                countsPerIndex.put(shardStats.getShardRouting().getIndex(), indexShardStats);
            }

            indexShardStats.total++;

            CommonStats shardCommonStats = shardStats.getStats();

            if (shardStats.getShardRouting().primary()) {
                indexShardStats.primaries++;
                docs.add(shardCommonStats.docs);
            }
            store.add(shardCommonStats.store);
            fieldData.add(shardCommonStats.fieldData);
            queryCache.add(shardCommonStats.queryCache);
            completion.add(shardCommonStats.completion);
            segments.add(shardCommonStats.segments);
            percolate.add(shardCommonStats.percolate);
        }
    }

    shards = new ShardStats();
    indexCount = countsPerIndex.size();
    for (ObjectObjectCursor<String, ShardStats> indexCountsCursor : countsPerIndex) {
        shards.addIndexShardCount(indexCountsCursor.value);
    }
}
 

public ObjectObjectHashMap<Double, IntFloatHashMap> createTfIdf(
    final BagOfPattern[] bagOfPatterns,
    final Set<Double> uniqueLabels) {
  int[] sampleIndices = createIndices(bagOfPatterns.length);
  return createTfIdf(bagOfPatterns, sampleIndices, uniqueLabels);
}
 

public TopicStats() {
    remotePublishersStats = new ObjectObjectHashMap<>();
    reset();
}
 

public NonPersistentTopicStats getStats(boolean getPreciseBacklog) {

        NonPersistentTopicStats stats = new NonPersistentTopicStats();

        ObjectObjectHashMap<String, PublisherStats> remotePublishersStats = new ObjectObjectHashMap<String, PublisherStats>();

        producers.values().forEach(producer -> {
            NonPersistentPublisherStats publisherStats = (NonPersistentPublisherStats) producer.getStats();
            stats.msgRateIn += publisherStats.msgRateIn;
            stats.msgThroughputIn += publisherStats.msgThroughputIn;

            if (producer.isRemote()) {
                remotePublishersStats.put(producer.getRemoteCluster(), publisherStats);
            } else {
                stats.getPublishers().add(publisherStats);
            }
        });

        stats.averageMsgSize = stats.msgRateIn == 0.0 ? 0.0 : (stats.msgThroughputIn / stats.msgRateIn);
        stats.msgInCounter = getMsgInCounter();
        stats.bytesInCounter = getBytesInCounter();

        subscriptions.forEach((name, subscription) -> {
            NonPersistentSubscriptionStats subStats = subscription.getStats();

            stats.msgRateOut += subStats.msgRateOut;
            stats.msgThroughputOut += subStats.msgThroughputOut;
            stats.bytesOutCounter += subStats.bytesOutCounter;
            stats.msgOutCounter += subStats.msgOutCounter;
            stats.getSubscriptions().put(name, subStats);
        });

        replicators.forEach((cluster, replicator) -> {
            NonPersistentReplicatorStats replicatorStats = replicator.getStats();

            // Add incoming msg rates
            PublisherStats pubStats = remotePublishersStats.get(replicator.getRemoteCluster());
            if (pubStats != null) {
                replicatorStats.msgRateIn = pubStats.msgRateIn;
                replicatorStats.msgThroughputIn = pubStats.msgThroughputIn;
                replicatorStats.inboundConnection = pubStats.getAddress();
                replicatorStats.inboundConnectedSince = pubStats.getConnectedSince();
            }

            stats.msgRateOut += replicatorStats.msgRateOut;
            stats.msgThroughputOut += replicatorStats.msgThroughputOut;

            stats.getReplication().put(replicator.getRemoteCluster(), replicatorStats);
        });

        return stats;
    }
 

public TopicStatsHelper() {
    remotePublishersStats = new ObjectObjectHashMap<>();
    reset();
}
 

public TopicStats getStats(boolean getPreciseBacklog) {

        TopicStats stats = new TopicStats();

        ObjectObjectHashMap<String, PublisherStats> remotePublishersStats = new ObjectObjectHashMap<String, PublisherStats>();

        producers.values().forEach(producer -> {
            PublisherStats publisherStats = producer.getStats();
            stats.msgRateIn += publisherStats.msgRateIn;
            stats.msgThroughputIn += publisherStats.msgThroughputIn;

            if (producer.isRemote()) {
                remotePublishersStats.put(producer.getRemoteCluster(), publisherStats);
            } else {
                stats.publishers.add(publisherStats);
            }
        });

        stats.averageMsgSize = stats.msgRateIn == 0.0 ? 0.0 : (stats.msgThroughputIn / stats.msgRateIn);
        stats.msgInCounter = getMsgInCounter();
        stats.bytesInCounter = getBytesInCounter();
        stats.msgChunkPublished = this.msgChunkPublished;

        subscriptions.forEach((name, subscription) -> {
            SubscriptionStats subStats = subscription.getStats(getPreciseBacklog);

            stats.msgRateOut += subStats.msgRateOut;
            stats.msgThroughputOut += subStats.msgThroughputOut;
            stats.bytesOutCounter += subStats.bytesOutCounter;
            stats.msgOutCounter += subStats.msgOutCounter;
            stats.subscriptions.put(name, subStats);
        });

        replicators.forEach((cluster, replicator) -> {
            ReplicatorStats replicatorStats = replicator.getStats();

            // Add incoming msg rates
            PublisherStats pubStats = remotePublishersStats.get(replicator.getRemoteCluster());
            if (pubStats != null) {
                replicatorStats.msgRateIn = pubStats.msgRateIn;
                replicatorStats.msgThroughputIn = pubStats.msgThroughputIn;
                replicatorStats.inboundConnection = pubStats.getAddress();
                replicatorStats.inboundConnectedSince = pubStats.getConnectedSince();
            }

            stats.msgRateOut += replicatorStats.msgRateOut;
            stats.msgThroughputOut += replicatorStats.msgThroughputOut;

            stats.replication.put(replicator.getRemoteCluster(), replicatorStats);
        });

        stats.storageSize = ledger.getTotalSize();
        stats.backlogSize = ledger.getEstimatedBacklogSize();
        stats.deduplicationStatus = messageDeduplication.getStatus().toString();

        return stats;
    }
 

public void registerContext(WriteContext context,ObjectObjectHashMap<KVPair, KVPair> indexToMainMutationMap){
    sharedMainMutationList.add(Pair.newPair(context,indexToMainMutationMap));
}