public static void addDoc(String doc, String updateRequestProcessorChain) throws Exception {
  Map<String, String[]> params = new HashMap<>();
  MultiMapSolrParams mmparams = new MultiMapSolrParams(params);
  params.put(UpdateParams.UPDATE_CHAIN, new String[]{updateRequestProcessorChain});
  SolrQueryRequestBase req = new SolrQueryRequestBase(h.getCore(),
      (SolrParams) mmparams) {
  };

  UpdateRequestHandler handler = new UpdateRequestHandler();
  handler.init(null);
  ArrayList<ContentStream> streams = new ArrayList<>(2);
  streams.add(new ContentStreamBase.StringStream(doc));
  req.setContentStreams(streams);
  handler.handleRequestBody(req, new SolrQueryResponse());
  req.close();
}
 

public ExtractingDocumentLoader(SolrQueryRequest req, UpdateRequestProcessor processor,
                         TikaConfig config, ParseContextConfig parseContextConfig,
                                SolrContentHandlerFactory factory) {
  this.params = req.getParams();
  this.core = req.getCore();
  this.config = config;
  this.parseContextConfig = parseContextConfig;
  this.processor = processor;

  templateAdd = new AddUpdateCommand(req);
  templateAdd.overwrite = params.getBool(UpdateParams.OVERWRITE, true);
  templateAdd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);

  //this is lightweight
  autoDetectParser = new AutoDetectParser(config);
  this.factory = factory;
  
  ignoreTikaException = params.getBool(ExtractingParams.IGNORE_TIKA_EXCEPTION, false);
}
 

@Test
public void testCommitWithin() throws Exception {
  DirectXmlRequest req = new DirectXmlRequest("/dataimport", xml);
  ModifiableSolrParams params = params("command", "full-import", 
      "clean", "false", UpdateParams.COMMIT, "false", 
      UpdateParams.COMMIT_WITHIN, "1000");
  req.setParams(params);
  try (HttpSolrClient solrServer = getHttpSolrClient(buildUrl(jetty.getLocalPort(), "/solr/collection1"))) {
    solrServer.request(req);
    Thread.sleep(100);
    ModifiableSolrParams queryAll = params("q", "*", "df", "desc");
    QueryResponse qres = solrServer.query(queryAll);
    SolrDocumentList results = qres.getResults();
    assertEquals(0, results.getNumFound());
    Thread.sleep(1000);
    for (int i = 0; i < 10; i++) {
      qres = solrServer.query(queryAll);
      results = qres.getResults();
      if (2 == results.getNumFound()) {
        return;
      }
      Thread.sleep(500);
    }
  }
  fail("Commit should have occurred but it did not");
}
 

private void testDistribOpenSearcher() {
  try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler)) {
    UpdateRequest updateRequest = new UpdateRequest();

    CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);

    //test default value (should be true)
    cmdDistrib.addCommit(updateRequest, ccmd);
    boolean openSearcher = updateRequest.getParams().getBool(UpdateParams.OPEN_SEARCHER, false);
    assertTrue(openSearcher);

    //test openSearcher = false
    ccmd.openSearcher = false;

    cmdDistrib.addCommit(updateRequest, ccmd);
    openSearcher = updateRequest.getParams().getBool(UpdateParams.OPEN_SEARCHER, true);
    assertFalse(openSearcher);
  }
}
 

private void addDoc(String doc) throws Exception {
  Map<String, String[]> params = new HashMap<>();
  MultiMapSolrParams mmparams = new MultiMapSolrParams(params);
  params.put(UpdateParams.UPDATE_CHAIN, new String[] { "uniq-fields" });
  SolrQueryRequestBase req = new SolrQueryRequestBase(h.getCore(),
      (SolrParams) mmparams) {
  };

  UpdateRequestHandler handler = new UpdateRequestHandler();
  handler.init(null);
  ArrayList<ContentStream> streams = new ArrayList<>(2);
  streams.add(new ContentStreamBase.StringStream(doc));
  req.setContentStreams(streams);
  handler.handleRequestBody(req, new SolrQueryResponse());
  req.close();
}
 

/** Specification of AtomicUpdateDocumentMerger is currently experimental.
 * @lucene.experimental
 */
public DistributedUpdateProcessor(SolrQueryRequest req,
    SolrQueryResponse rsp, AtomicUpdateDocumentMerger docMerger,
    UpdateRequestProcessor next) {
  super(next);
  this.rsp = rsp;
  this.docMerger = docMerger;
  this.idField = req.getSchema().getUniqueKeyField();
  this.req = req;
  this.replicaType = computeReplicaType();
  // version init

  this.ulog = req.getCore().getUpdateHandler().getUpdateLog();
  this.vinfo = ulog == null ? null : ulog.getVersionInfo();
  versionsStored = this.vinfo != null && this.vinfo.getVersionField() != null;
  returnVersions = req.getParams().getBool(UpdateParams.VERSIONS ,false);

  // TODO: better way to get the response, or pass back info to it?
  // SolrRequestInfo reqInfo = returnVersions ? SolrRequestInfo.getRequestInfo() : null;

  // this should always be used - see filterParams
  DistributedUpdateProcessorFactory.addParamToDistributedRequestWhitelist
    (this.req, UpdateParams.UPDATE_CHAIN, TEST_DISTRIB_SKIP_SERVERS, CommonParams.VERSION_FIELD,
        UpdateParams.EXPUNGE_DELETES, UpdateParams.OPTIMIZE, UpdateParams.MAX_OPTIMIZE_SEGMENTS, ShardParams._ROUTE_);

  //this.rsp = reqInfo != null ? reqInfo.getRsp() : null;
}
 

SingleThreadedJsonLoader(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor processor) {
  this.processor = processor;
  this.req = req;
  this.rsp = rsp;

  commitWithin = req.getParams().getInt(UpdateParams.COMMIT_WITHIN, -1);
  overwrite = req.getParams().getBool(UpdateParams.OVERWRITE, true);
}
 

private void delete(SolrQueryRequest req, UpdateRequest update, UpdateRequestProcessor processor) throws IOException {
  SolrParams params = update.getParams();
  DeleteUpdateCommand delcmd = new DeleteUpdateCommand(req);
  if(params != null) {
    delcmd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
  }
  
  if(update.getDeleteByIdMap() != null) {
    Set<Entry<String,Map<String,Object>>> entries = update.getDeleteByIdMap().entrySet();
    for (Entry<String,Map<String,Object>> e : entries) {
      delcmd.id = e.getKey();
      Map<String,Object> map = e.getValue();
      if (map != null) {
        Long version = (Long) map.get("ver");
        if (version != null) {
          delcmd.setVersion(version);
        }
      }
      if (map != null) {
        String route = (String) map.get(ShardParams._ROUTE_);
        if (route != null) {
          delcmd.setRoute(route);
        }
      }
      processor.processDelete(delcmd);
      delcmd.clear();
    }
  }
  
  if(update.getDeleteQuery() != null) {
    for (String s : update.getDeleteQuery()) {
      delcmd.query = s;
      processor.processDelete(delcmd);
    }
  }
}
 

private void commitOnLeader(String leaderUrl) throws SolrServerException,
    IOException {
  try (HttpSolrClient client = new HttpSolrClient.Builder(leaderUrl)
      .withConnectionTimeout(30000)
      .build()) {
    UpdateRequest ureq = new UpdateRequest();
    ureq.setParams(new ModifiableSolrParams());
    ureq.getParams().set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    ureq.getParams().set(UpdateParams.OPEN_SEARCHER, false);
    ureq.setAction(AbstractUpdateRequest.ACTION.COMMIT, false, true).process(
        client);
  }
}
 

/**
 * Modify UpdateCommand based on request parameters
 */
public static void updateCommit(CommitUpdateCommand cmd, SolrParams params) {
  if( params == null ) return;

  cmd.openSearcher = params.getBool( UpdateParams.OPEN_SEARCHER, cmd.openSearcher );
  cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, cmd.waitSearcher );
  cmd.softCommit = params.getBool( UpdateParams.SOFT_COMMIT, cmd.softCommit );
  cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES, cmd.expungeDeletes );
  cmd.maxOptimizeSegments = params.getInt( UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments );
  cmd.prepareCommit = params.getBool( UpdateParams.PREPARE_COMMIT,   cmd.prepareCommit );
}
 

@Override
public void load(SolrQueryRequest req, SolrQueryResponse rsp,
    ContentStream stream, UpdateRequestProcessor processor) throws Exception {

  ContentStreamLoader loader = pathVsLoaders.get(req.getContext().get(PATH));
  if(loader == null) {
    String type = req.getParams().get(UpdateParams.ASSUME_CONTENT_TYPE);
    if (type == null) {
      type = stream.getContentType();
    }
    if (type == null) { // Normal requests will not get here.
      throw new SolrException(ErrorCode.UNSUPPORTED_MEDIA_TYPE, "Missing ContentType");
    }
    int idx = type.indexOf(';');
    if (idx > 0) {
      type = type.substring(0, idx);
    }
    loader = loaders.get(type);
    if (loader == null) {
      throw new SolrException(ErrorCode.UNSUPPORTED_MEDIA_TYPE, "Unsupported ContentType: "
          + type + "  Not in: " + loaders.keySet());
    }
  }

  if(loader.getDefaultWT()!=null) {
    setDefaultWT(req,loader);
  }
  loader.load(req, rsp, stream, processor);
}
 

final private void commitOnLeader(String leaderUrl) throws SolrServerException,
    IOException {
  try (HttpSolrClient client = buildRecoverySolrClient(leaderUrl)) {
    UpdateRequest ureq = new UpdateRequest();
    ureq.setParams(new ModifiableSolrParams());
    // ureq.getParams().set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    // ureq.getParams().set(UpdateParams.OPEN_SEARCHER, onlyLeaderIndexes);// Why do we need to open searcher if
    // "onlyLeaderIndexes"?
    ureq.getParams().set(UpdateParams.OPEN_SEARCHER, false);
    ureq.setAction(AbstractUpdateRequest.ACTION.COMMIT, false, true).process(
        client);
  }
}
 

public UpdateRequestProcessorChain getUpdateProcessorChain(SolrParams params) {
  String chainName = params.get(UpdateParams.UPDATE_CHAIN);
  UpdateRequestProcessorChain defaultUrp = getUpdateProcessingChain(chainName);
  ProcessorInfo processorInfo = new ProcessorInfo(params);
  if (processorInfo.isEmpty()) return defaultUrp;
  return UpdateRequestProcessorChain.constructChain(defaultUrp, processorInfo, this);
}
 

private AddUpdateCommand getAddCommand(final SolrQueryRequest req) {
	final AddUpdateCommand addCmd = new AddUpdateCommand(req);
	
	addCmd.overwrite = req.getParams().getBool(UpdateParams.OVERWRITE, true);
	addCmd.commitWithin = req.getParams().getInt(UpdateParams.COMMIT_WITHIN, -1);
	return addCmd;
}
 

static void addDoc(String doc, String chain) throws Exception {
	Map<String, String[]> params = new HashMap<>();
	MultiMapSolrParams mmparams = new MultiMapSolrParams(params);
	params.put(UpdateParams.UPDATE_CHAIN, new String[] { chain });
	SolrQueryRequestBase req = new SolrQueryRequestBase(h.getCore(), mmparams) {
	};

	UpdateRequestHandler handler = new UpdateRequestHandler();
	handler.init(null);
	ArrayList<ContentStream> streams = new ArrayList<>(2);
	streams.add(new ContentStreamBase.StringStream(doc));
	req.setContentStreams(streams);
	handler.handleRequestBody(req, new SolrQueryResponse());
	req.close();
}
 

public AbstractUpdateRequest setAction(ACTION action, boolean waitFlush, boolean waitSearcher, int maxSegments , boolean softCommit, boolean expungeDeletes) {
  setAction(action, waitFlush, waitSearcher,softCommit,maxSegments) ;
  params.set(UpdateParams.EXPUNGE_DELETES, String.valueOf(expungeDeletes));
  return this;
}
 

public AbstractUpdateRequest setAction(ACTION action, boolean waitFlush, boolean waitSearcher, int maxSegments, boolean softCommit, boolean expungeDeletes, boolean openSearcher) {
  setAction(action, waitFlush, waitSearcher, maxSegments, softCommit, expungeDeletes);
  params.set(UpdateParams.OPEN_SEARCHER, String.valueOf(openSearcher));
  return this;
}
 

public boolean isWaitSearcher() {
  return params != null && params.getBool(UpdateParams.WAIT_SEARCHER, false);
}
 

public ACTION getAction() {
  if (params==null) return null;
  if (params.getBool(UpdateParams.COMMIT, false)) return ACTION.COMMIT;
  if (params.getBool(UpdateParams.OPTIMIZE, false)) return ACTION.OPTIMIZE;
  return null;
}
 

public void setWaitSearcher(boolean waitSearcher) {
  setParam( UpdateParams.WAIT_SEARCHER, waitSearcher+"" );
}
 

@Test
public void testVersionsAreReturned() throws Exception {
  CollectionAdminRequest.createCollection("versions_collection", "conf", 2, 1).process(cluster.getSolrClient());
  cluster.waitForActiveCollection("versions_collection", 2, 2);
  
  // assert that "adds" are returned
  UpdateRequest updateRequest = new UpdateRequest()
      .add("id", "1", "a_t", "hello1")
      .add("id", "2", "a_t", "hello2");
  updateRequest.setParam(UpdateParams.VERSIONS, Boolean.TRUE.toString());

  NamedList<Object> response = updateRequest.commit(getRandomClient(), "versions_collection").getResponse();
  Object addsObject = response.get("adds");
  
  assertNotNull("There must be a adds parameter", addsObject);
  assertTrue(addsObject instanceof NamedList<?>);
  NamedList<?> adds = (NamedList<?>) addsObject;
  assertEquals("There must be 2 versions (one per doc)", 2, adds.size());

  Map<String, Long> versions = new HashMap<>();
  Object object = adds.get("1");
  assertNotNull("There must be a version for id 1", object);
  assertTrue("Version for id 1 must be a long", object instanceof Long);
  versions.put("1", (Long) object);

  object = adds.get("2");
  assertNotNull("There must be a version for id 2", object);
  assertTrue("Version for id 2 must be a long", object instanceof Long);
  versions.put("2", (Long) object);

  QueryResponse resp = getRandomClient().query("versions_collection", new SolrQuery("*:*"));
  assertEquals("There should be one document because overwrite=true", 2, resp.getResults().getNumFound());

  for (SolrDocument doc : resp.getResults()) {
    Long version = versions.get(doc.getFieldValue("id"));
    assertEquals("Version on add must match _version_ field", version, doc.getFieldValue("_version_"));
  }

  // assert that "deletes" are returned
  UpdateRequest deleteRequest = new UpdateRequest().deleteById("1");
  deleteRequest.setParam(UpdateParams.VERSIONS, Boolean.TRUE.toString());
  response = deleteRequest.commit(getRandomClient(), "versions_collection").getResponse();
  Object deletesObject = response.get("deletes");
  assertNotNull("There must be a deletes parameter", deletesObject);
  @SuppressWarnings({"rawtypes"})
  NamedList deletes = (NamedList) deletesObject;
  assertEquals("There must be 1 version", 1, deletes.size());
}
 

@Test
public void testVersionsAreReturned() throws Exception {
  CollectionAdminRequest.createCollection("versions_collection", "conf", 2, 1).process(cluster.getSolrClient());
  cluster.waitForActiveCollection("versions_collection", 2, 2);
  
  // assert that "adds" are returned
  UpdateRequest updateRequest = new UpdateRequest()
      .add("id", "1", "a_t", "hello1")
      .add("id", "2", "a_t", "hello2");
  updateRequest.setParam(UpdateParams.VERSIONS, Boolean.TRUE.toString());

  NamedList<Object> response = updateRequest.commit(getRandomClient(), "versions_collection").getResponse();
  Object addsObject = response.get("adds");
  
  assertNotNull("There must be a adds parameter", addsObject);
  assertTrue(addsObject instanceof NamedList<?>);
  NamedList<?> adds = (NamedList<?>) addsObject;
  assertEquals("There must be 2 versions (one per doc)", 2, adds.size());

  Map<String, Long> versions = new HashMap<>();
  Object object = adds.get("1");
  assertNotNull("There must be a version for id 1", object);
  assertTrue("Version for id 1 must be a long", object instanceof Long);
  versions.put("1", (Long) object);

  object = adds.get("2");
  assertNotNull("There must be a version for id 2", object);
  assertTrue("Version for id 2 must be a long", object instanceof Long);
  versions.put("2", (Long) object);

  QueryResponse resp = getRandomClient().query("versions_collection", new SolrQuery("*:*"));
  assertEquals("There should be one document because overwrite=true", 2, resp.getResults().getNumFound());

  for (SolrDocument doc : resp.getResults()) {
    Long version = versions.get(doc.getFieldValue("id"));
    assertEquals("Version on add must match _version_ field", version, doc.getFieldValue("_version_"));
  }

  // assert that "deletes" are returned
  UpdateRequest deleteRequest = new UpdateRequest().deleteById("1");
  deleteRequest.setParam(UpdateParams.VERSIONS, Boolean.TRUE.toString());
  response = deleteRequest.commit(getRandomClient(), "versions_collection").getResponse();
  Object deletesObject = response.get("deletes");
  assertNotNull("There must be a deletes parameter", deletesObject);
  @SuppressWarnings({"rawtypes"})
  NamedList deletes = (NamedList) deletesObject;
  assertEquals("There must be 1 version", 1, deletes.size());
}
 

public SolrWriter(UpdateRequestProcessor processor, SolrQueryRequest req) {
  this.processor = processor;
  this.req = req;
  commitWithin = (req != null) ? req.getParams().getInt(UpdateParams.COMMIT_WITHIN, -1): -1;
}
 

CSVLoaderBase(SolrQueryRequest req, UpdateRequestProcessor processor) {
  this.processor = processor;
  this.params = req.getParams();
  this.literals = new HashMap<>();

  templateAdd = new AddUpdateCommand(req);
  templateAdd.overwrite=params.getBool(OVERWRITE,true);
  templateAdd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
  
  strategy = new CSVStrategy(',', '"', CSVStrategy.COMMENTS_DISABLED, CSVStrategy.ESCAPE_DISABLED, false, false, false, true, "\n");
  String sep = params.get(SEPARATOR);
  if (sep!=null) {
    if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid separator:'"+sep+"'");
    strategy.setDelimiter(sep.charAt(0));
  }

  String encapsulator = params.get(ENCAPSULATOR);
  if (encapsulator!=null) {
    if (encapsulator.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid encapsulator:'"+encapsulator+"'");
  }

  String escape = params.get(ESCAPE);
  if (escape!=null) {
    if (escape.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid escape:'"+escape+"'");
  }
  rowId = params.get(ROW_ID);
  rowIdOffset = params.getInt(ROW_ID_OFFSET, 0);

  // if only encapsulator or escape is set, disable the other escaping mechanism
  if (encapsulator == null && escape != null) {
    strategy.setEncapsulator( CSVStrategy.ENCAPSULATOR_DISABLED);     
    strategy.setEscape(escape.charAt(0));
  } else {
    if (encapsulator != null) {
      strategy.setEncapsulator(encapsulator.charAt(0));
    }
    if (escape != null) {
      char ch = escape.charAt(0);
      strategy.setEscape(ch);
      if (ch == '\\') {
        // If the escape is the standard backslash, then also enable
        // unicode escapes (it's harmless since 'u' would not otherwise
        // be escaped.                    
        strategy.setUnicodeEscapeInterpretation(true);
      }
    }
  }

  String fn = params.get(FIELDNAMES);
  fieldnames = fn != null ? commaSplit.split(fn,-1) : null;

  Boolean hasHeader = params.getBool(HEADER);

  skipLines = params.getInt(SKIPLINES,0);

  if (fieldnames==null) {
    if (null == hasHeader) {
      // assume the file has the headers if they aren't supplied in the args
      hasHeader=true;
    } else if (!hasHeader) {
      throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"CSVLoader: must specify fieldnames=<fields>* or header=true");
    }
  } else {
    // if the fieldnames were supplied and the file has a header, we need to
    // skip over that header.
    if (hasHeader!=null && hasHeader) skipLines++;

    prepareFields();
  }
}
 

private AddUpdateCommand getAddCommand(SolrQueryRequest req, SolrParams params) {
  AddUpdateCommand addCmd = new AddUpdateCommand(req);
  addCmd.overwrite = params.getBool(UpdateParams.OVERWRITE, true);
  addCmd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);
  return addCmd;
}
 

@Override
public void load(
		final SolrQueryRequest request, 
		final SolrQueryResponse response,
		final ContentStream stream, 
		final UpdateRequestProcessor processor) throws Exception {
	
	// Default ContentStream implementation starts reading the stream and
	// if it starts with '<' then it assumes a content type of "application/xml", 
	// if it starts with '{' then it assumes a content type of "application/json" 			
	// This behaviour is wrong is SolRDF and maybe we need a custom ContentStream here
	// At the moment this is just a workaround:
	final String contentType = stream.getContentType() != null 
			&& !"application/xml".equals(stream.getContentType())
			&& !"application/json".equals(stream.getContentType()) 
				? stream.getContentType() 
				: request.getParams().get(UpdateParams.ASSUME_CONTENT_TYPE);
	
	log.debug(MessageCatalog._00094_BULK_LOADER_CT, contentType);			
				
	final Lang lang = RDFLanguages.contentTypeToLang(contentType);
	if (lang == null) {
		final String message = MessageFactory.createMessage(MessageCatalog._00095_INVALID_CT, contentType);
		log.error(message);							
		throw new SolrException(ErrorCode.BAD_REQUEST, message);
	}
	
	final ContentStreamLoader delegate = 
			(lang == Lang.NQ || lang == Lang.NQUADS || lang == Lang.TRIG)
				? quadsLoader
				: triplesLoader;
	
	log.debug(MessageCatalog._00096_SELECTED_BULK_LOADER, contentType, delegate);
	
	delegate.load(
			request, 
			response, 
			new ContentStream() {	
				@Override
				public InputStream getStream() throws IOException {
					return stream.getStream();
				}
				
				@Override
				public String getSourceInfo() {
					return stream.getSourceInfo();
				}
				
				@Override
				public Long getSize() {
					return stream.getSize();
				}
				
				@Override
				public Reader getReader() throws IOException {
					return stream.getReader();
				}
				
				@Override
				public String getName() {
					return stream.getName();
				}
				
				@Override
				public String getContentType() {
					return contentType;
				}
			}, 
		processor);
}
 

/**
 * Gets the processor build from the processor update chain(UpdateParams.UPDATE_CHAIN)
 *
 * @param req - the solr query request information
 * @param rsp - the solr query response information
 * @return the update request processor
 */
private UpdateRequestProcessor getProcessor(SolrQueryRequest req, SolrQueryResponse rsp) {
    UpdateRequestProcessorChain processorChain =
            req.getCore().getUpdateProcessingChain(req.getParams().get(UpdateParams.UPDATE_CHAIN));
    return processorChain.createProcessor(req, rsp);
}