@BeforeClass
public static void setupCluster() throws Exception {
  System.setProperty("solr.zkclienttimeout", "20000");

  configureCluster(4)
      .addConfig("conf", configset("cloud-minimal"))
      .configure();
  // Add proxies
  proxies = new HashMap<>(cluster.getJettySolrRunners().size());
  jettys = new HashMap<>(cluster.getJettySolrRunners().size());
  for (JettySolrRunner jetty:cluster.getJettySolrRunners()) {
    SocketProxy proxy = new SocketProxy();
    jetty.setProxyPort(proxy.getListenPort());
    cluster.stopJettySolrRunner(jetty);//TODO: Can we avoid this restart
    cluster.startJettySolrRunner(jetty);
    cluster.waitForAllNodes(30);
    proxy.open(jetty.getBaseUrl().toURI());
    if (log.isInfoEnabled()) {
      log.info("Adding proxy for URL: {}. Proxy: {}", jetty.getBaseUrl(), proxy.getUrl());
    }
    proxies.put(proxy.getUrl(), proxy);
    jettys.put(proxy.getUrl(), jetty);
  }
}
 

public static JettySolrRunner createAndStartJetty(String solrHome, String configFile, String schemaFile, String context,
                                          boolean stopAtShutdown, SortedMap<ServletHolder,String> extraServlets)
    throws Exception {
  // creates the data dir

  context = context==null ? "/solr" : context;
  SolrJettyTestBase.context = context;

  JettyConfig jettyConfig = JettyConfig.builder()
      .setContext(context)
      .stopAtShutdown(stopAtShutdown)
      .withServlets(extraServlets)
      .withSSLConfig(sslConfig.buildServerSSLConfig())
      .build();

  Properties nodeProps = new Properties();
  if (configFile != null)
    nodeProps.setProperty("solrconfig", configFile);
  if (schemaFile != null)
    nodeProps.setProperty("schema", schemaFile);
  if (System.getProperty("solr.data.dir") == null && System.getProperty("solr.hdfs.home") == null) {
    nodeProps.setProperty("solr.data.dir", createTempDir().toFile().getCanonicalPath());
  }

  return createAndStartJetty(solrHome, nodeProps, jettyConfig);
}
 

@Test
public void testCreateNodeSet() throws Exception {
  JettySolrRunner jetty1 = cluster.getRandomJetty(random());
  JettySolrRunner jetty2 = cluster.getRandomJetty(random());

  List<String> baseUrls = ImmutableList.of(jetty1.getBaseUrl().toString(), jetty2.getBaseUrl().toString());

  CollectionAdminRequest.createCollection("nodeset_collection", "conf", 2, 1)
      .setCreateNodeSet(baseUrls.get(0) + "," + baseUrls.get(1))
      .process(cluster.getSolrClient());

  DocCollection collectionState = getCollectionState("nodeset_collection");
  for (Replica replica : collectionState.getReplicas()) {
    String replicaUrl = replica.getCoreUrl();
    boolean matchingJetty = false;
    for (String jettyUrl : baseUrls) {
      if (replicaUrl.startsWith(jettyUrl)) {
        matchingJetty = true;
      }
    }
    if (matchingJetty == false) {
      fail("Expected replica to be on " + baseUrls + " but was on " + replicaUrl);
    }
  }
}
 

public static void start(List<JettySolrRunner> jettys) throws Exception {
  ExecutorService executor = new ExecutorUtil.MDCAwareThreadPoolExecutor(
      0,
      Integer.MAX_VALUE,
      15, TimeUnit.SECONDS,
      new SynchronousQueue<>(),
      new SolrNamedThreadFactory("ChaosMonkey"),
      false);
  for (JettySolrRunner jetty : jettys) {
    executor.submit(() -> {
      try {
        jetty.start();
      } catch (Exception e) {
        log.error("error starting jetty", e);
        throw new RuntimeException(e);
      }
    });
  }
  ExecutorUtil.shutdownAndAwaitTermination(executor);
}
 

private List<SolrCore> getSolrCore(boolean isLeader) {
  List<SolrCore> rs = new ArrayList<>();

  CloudSolrClient cloudClient = cluster.getSolrClient();
  DocCollection docCollection = cloudClient.getZkStateReader().getClusterState().getCollection(collectionName);

  for (JettySolrRunner solrRunner : cluster.getJettySolrRunners()) {
    if (solrRunner.getCoreContainer() == null) continue;
    for (SolrCore solrCore : solrRunner.getCoreContainer().getCores()) {
      CloudDescriptor cloudDescriptor = solrCore.getCoreDescriptor().getCloudDescriptor();
      Slice slice = docCollection.getSlice(cloudDescriptor.getShardId());
      Replica replica = docCollection.getReplica(cloudDescriptor.getCoreNodeName());
      if (slice.getLeader().equals(replica) && isLeader) {
        rs.add(solrCore);
      } else if (!slice.getLeader().equals(replica) && !isLeader) {
        rs.add(solrCore);
      }
    }
  }
  return rs;
}
 

/**
 * General purpose cluster sanity check...
 * <ol>
 * <li>Upload a config set</li>
 * <li>verifies a collection can be created</li>
 * <li>verifies many things that should succeed/fail when communicating with the cluster according to the specified sslConfig</li>
 * <li>shutdown a server &amp; startup a new one in it's place</li>
 * <li>repeat the verifications of ssl / no-ssl communication</li>
 * <li>create a second collection</li>
 * </ol>
 * @see #CONF_NAME
 * @see #NUM_SERVERS
 */
public static void checkClusterWithCollectionCreations(final MiniSolrCloudCluster cluster,
                                                       final SSLTestConfig sslConfig) throws Exception {

  cluster.uploadConfigSet(SolrTestCaseJ4.TEST_PATH().resolve("collection1").resolve("conf"), CONF_NAME);
  
  checkCreateCollection(cluster, "first_collection");
  
  checkClusterJettys(cluster, sslConfig);
  
  // shut down a server
  JettySolrRunner stoppedServer = cluster.stopJettySolrRunner(0);
  cluster.waitForJettyToStop(stoppedServer);
  assertTrue(stoppedServer.isStopped());
  assertEquals(NUM_SERVERS - 1, cluster.getJettySolrRunners().size());
  
  // create a new server
  JettySolrRunner startedServer = cluster.startJettySolrRunner();
  cluster.waitForAllNodes(30);
  assertTrue(startedServer.isRunning());
  assertEquals(NUM_SERVERS, cluster.getJettySolrRunners().size());
  
  checkClusterJettys(cluster, sslConfig);
  
  checkCreateCollection(cluster, "second_collection");
}
 

@Override
public void tearDown() throws Exception {
  for (JettySolrRunner jetty:cluster.getJettySolrRunners()) {
    if (!jetty.isRunning()) {
      log.warn("Jetty {} not running, probably some bad test. Starting it", jetty.getLocalPort());
      jetty.start();
    }
  }
  if (cluster.getSolrClient().getZkStateReader().getClusterState().getCollectionOrNull(collectionName) != null) {
    log.info("tearDown deleting collection");
    CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
    log.info("Collection deleted");
    waitForDeletion(collectionName);
  }
  super.tearDown();
}
 

@Test
public void testErrorsInShutdown() throws Exception {

  AtomicInteger jettyIndex = new AtomicInteger();

  MiniSolrCloudCluster cluster = new MiniSolrCloudCluster(3, createTempDir(), JettyConfig.builder().build()) {
    @Override
    public JettySolrRunner stopJettySolrRunner(JettySolrRunner jetty) throws Exception {
      JettySolrRunner j = super.stopJettySolrRunner(jetty);
      if (jettyIndex.incrementAndGet() == 2)
        throw new IOException("Fake IOException on shutdown!");
      return j;
    }
  };

  Exception ex = expectThrows(Exception.class, cluster::shutdown);
  assertEquals("Error shutting down MiniSolrCloudCluster", ex.getMessage());
  assertEquals("Expected one suppressed exception", 1, ex.getSuppressed().length);
  assertEquals("Fake IOException on shutdown!", ex.getSuppressed()[0].getMessage());
}
 

private void bringBackOldLeaderAndSendDoc(String collection, Replica leader, List<Replica> notLeaders, int docid) throws Exception {
  // Bring back the leader which was stopped
  log.info("Bringing back originally killed leader...");
  JettySolrRunner leaderJetty = getJettyOnPort(getReplicaPort(leader));
  getProxyForReplica(leader).reopen();
  leaderJetty.start();
  waitForRecoveriesToFinish(collection, cloudClient.getZkStateReader(), true);
  cloudClient.getZkStateReader().forceUpdateCollection(collection);
  ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
  if (log.isInfoEnabled()) {
    log.info("After bringing back leader: {}", clusterState.getCollection(collection).getSlice(SHARD1));
  }
  int numActiveReplicas = getNumberOfActiveReplicas(clusterState, collection, SHARD1);
  assertEquals(1+notLeaders.size(), numActiveReplicas);
  log.info("Sending doc {}...", docid);
  sendDoc(docid);
  log.info("Committing...");
  cloudClient.commit();
  log.info("Doc {} sent and commit issued", docid);
  assertDocsExistInAllReplicas(notLeaders, collection, docid, docid);
  assertDocsExistInAllReplicas(Collections.singletonList(leader), collection, docid, docid);
}
 

@Test
@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/HADOOP-9893")
public void testForwarding() throws Exception {
  String collectionName = "forwardingCollection";

  // create collection
  CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1",
      1, 1);
  try (SolrClient solrClient = newSolrClient()) {
    create.process(solrClient);
  }

  // try a command to each node, one of them must be forwarded
  for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
    try (HttpSolrClient client =
             new HttpSolrClient.Builder(jetty.getBaseUrl().toString() + "/" + collectionName).build()) {
      ModifiableSolrParams params = new ModifiableSolrParams();
      params.set("q", "*:*");
      params.set(USER_PARAM, "user");
      client.query(params);
    }
  }
}
 

public static void initSingleSolrServer(String testClassName, Properties solrcoreProperties) throws Throwable
{
    initSolrServers(0,testClassName,solrcoreProperties);
    
    JettySolrRunner jsr = jettyContainers.get(testClassName);
    CoreContainer coreContainer = jsr.getCoreContainer();
    AlfrescoCoreAdminHandler coreAdminHandler = (AlfrescoCoreAdminHandler)  coreContainer.getMultiCoreHandler();
    assertNotNull(coreAdminHandler);
    String[] extras = null;
    if ((solrcoreProperties != null) && !solrcoreProperties.isEmpty())
    {
        int i = 0;
        extras = new String[solrcoreProperties.size()*2];
        for (Map.Entry<Object, Object> prop:solrcoreProperties.entrySet())
        {
            extras[i++] = "property."+prop.getKey();
            extras[i++] = (String) prop.getValue();
        }

    }
    defaultCore = createCoreUsingTemplate(coreContainer, coreAdminHandler, "alfresco", "rerank", 1, 1, extras);
    assertNotNull(defaultCore);
    String url = buildUrl(jsr.getLocalPort()) + "/" + "alfresco";
    SolrClient standaloneClient = createNewSolrClient(url);
    assertNotNull(standaloneClient);
    solrCollectionNameToStandaloneClient.put("alfresco", standaloneClient);
}
 

/**
 * Creates a JettySolrRunner (if one didn't exist already). DOES NOT START IT.
 */
protected static JettySolrRunner createJetty(String jettyKey, boolean basicAuth) throws Exception
{
    if (jettyContainers.containsKey(jettyKey))
    {
        return jettyContainers.get(jettyKey);
    }
    else
    {
        Path jettySolrHome = testDir.toPath().resolve(jettyKey);
        seedSolrHome(jettySolrHome);
        return createJetty(jettySolrHome.toFile(), null, null, false, 0, getSchemaFile(), basicAuth);
    }
}
 

/**
 * Starts jetty if its not already running
 */
protected static void start(JettySolrRunner jsr) throws Exception
{
    if (!jsr.isRunning())
    {
        jsr.start();
    }
}
 

@Before
public void setupCluster() throws Exception {
  TestInjection.skipIndexWriterCommitOnClose = TEST_VALUE_FOR_SKIP_COMMIT_ON_CLOSE;
  
  System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
  System.setProperty("solr.ulog.numRecordsToKeep", "1000");
  System.setProperty("leaderVoteWait", "60000");

  configureCluster(2)
      .addConfig("conf", configset("cloud-minimal"))
      .configure();

  NODE0 = cluster.getJettySolrRunner(0);
  NODE1 = cluster.getJettySolrRunner(1);
    
  // Add proxies
  proxies = new HashMap<>(cluster.getJettySolrRunners().size());
  jettys = new HashMap<>();
  for (JettySolrRunner jetty:cluster.getJettySolrRunners()) {
    SocketProxy proxy = new SocketProxy();
    jetty.setProxyPort(proxy.getListenPort());
    cluster.stopJettySolrRunner(jetty);//TODO: Can we avoid this restart
    cluster.startJettySolrRunner(jetty);
    proxy.open(jetty.getBaseUrl().toURI());
    if (log.isInfoEnabled()) {
      log.info("Adding proxy for URL: {}. Proxy: {}", jetty.getBaseUrl(), proxy.getUrl());
    }
    proxies.put(jetty, proxy);
    jettys.put(proxy.getUrl(), jetty);
  }
}
 

/**
 * Overrides the parent implementation to install a SocketProxy in-front of the Jetty server.
 */
@Override
public JettySolrRunner createJetty(File solrHome, String dataDir,
                                   String shardList, String solrConfigOverride, String schemaOverride, Replica.Type replicaType)
    throws Exception {
  return createProxiedJetty(solrHome, dataDir, shardList, solrConfigOverride, schemaOverride, replicaType);
}
 

protected void waitCoreCount(String collection, int count) {
  long start = System.nanoTime();
  int coreFooCount;
  List<JettySolrRunner> jsrs = cluster.getJettySolrRunners();
  do {
    coreFooCount = 0;
    // have to check all jetties... there was a very confusing bug where we only checked one and
    // thus might pick a jetty without a core for the collection and succeed if count = 0 when we
    // should have failed, or at least waited longer
    for (JettySolrRunner jsr : jsrs) {
      List<CoreDescriptor> coreDescriptors = jsr.getCoreContainer().getCoreDescriptors();
      for (CoreDescriptor coreDescriptor : coreDescriptors) {
        String collectionName = coreDescriptor.getCollectionName();
        if (collection.equals(collectionName)) {
          coreFooCount ++;
        }
      }
    }
    if (NANOSECONDS.toSeconds(System.nanoTime() - start) > 60) {
      fail("took over 60 seconds after collection creation to update aliases:"+collection + " core count=" + coreFooCount + " was looking for " + count);
    } else {
      try {
        Thread.sleep(500);
      } catch (InterruptedException e) {
        e.printStackTrace();
        fail(e.getMessage());
      }
    }
  } while(coreFooCount != count);
}
 

@Test
public void testWaitForStateWatcherIsRetainedOnPredicateFailure() throws Exception {

  CloudSolrClient client = cluster.getSolrClient();
  CollectionAdminRequest.createCollection("falsepredicate", "config", 4, 1)
    .processAndWait(client, MAX_WAIT_TIMEOUT);

  client.waitForState("falsepredicate", MAX_WAIT_TIMEOUT, TimeUnit.SECONDS,
                      (n, c) -> DocCollection.isFullyActive(n, c, 4, 1));

  final CountDownLatch firstCall = new CountDownLatch(1);

  // stop a node, then add a watch waiting for all nodes to be back up
  JettySolrRunner node1 = cluster.stopJettySolrRunner(random().nextInt
                                                      (cluster.getJettySolrRunners().size()));
  
  cluster.waitForJettyToStop(node1);

  Future<Boolean> future = waitInBackground("falsepredicate", MAX_WAIT_TIMEOUT, TimeUnit.SECONDS,
                                            (liveNodes, collectionState) -> {
        firstCall.countDown();
        return DocCollection.isFullyActive(liveNodes, collectionState, 4, 1);
      });

  // first, stop another node; the watch should not be fired after this!
  JettySolrRunner node2 = cluster.stopJettySolrRunner(random().nextInt
                                                      (cluster.getJettySolrRunners().size()));

  // now start them both back up
  cluster.startJettySolrRunner(node1);
  assertTrue("CollectionStateWatcher not called",
             firstCall.await(MAX_WAIT_TIMEOUT, TimeUnit.SECONDS));
  cluster.startJettySolrRunner(node2);

  Boolean result = future.get();
  assertTrue("Did not see a fully active cluster", result);

}
 

private void populateSolrUrls(MiniSolrCloudCluster cluster) {
  if (random().nextBoolean()) {
    final List<JettySolrRunner> solrNodes = cluster.getJettySolrRunners();
    for (JettySolrRunner node : solrNodes) {
      this.solrUrls.add(node.getBaseUrl().toString());
    }
  } else {
    this.solrUrls.add(cluster.getRandomJetty(random()).getBaseUrl().toString());
  }
}
 

private void checkForSingleIndex(JettySolrRunner jetty, boolean afterReload) {
  CoreContainer cores = jetty.getCoreContainer();
  Collection<SolrCore> theCores = cores.getCores();
  for (SolrCore core : theCores) {
    String ddir = core.getDataDir();
    CachingDirectoryFactory dirFactory = getCachingDirectoryFactory(core);
    synchronized (dirFactory) {
      Set<String> livePaths = dirFactory.getLivePaths();
      // one for data, one for the index under data and one for the snapshot metadata.
      // we also allow one extra index dir - it may not be removed until the core is closed
      if (afterReload) {
        assertTrue(livePaths.toString() + ":" + livePaths.size(), 3 == livePaths.size() || 4 == livePaths.size());
      } else {
        assertTrue(livePaths.toString() + ":" + livePaths.size(), 3 == livePaths.size());
      }

      // :TODO: assert that one of the paths is a subpath of hte other
    }
    if (dirFactory instanceof StandardDirectoryFactory) {
      System.out.println(Arrays.asList(new File(ddir).list()));
      // we also allow one extra index dir - it may not be removed until the core is closed
      int cnt = indexDirCount(ddir);
      // if after reload, there may be 2 index dirs while the reloaded SolrCore closes.
      if (afterReload) {
        assertTrue("found:" + cnt + Arrays.asList(new File(ddir).list()).toString(), 1 == cnt || 2 == cnt);
      } else {
        assertTrue("found:" + cnt + Arrays.asList(new File(ddir).list()).toString(), 1 == cnt);
      }

    }
  }
}
 

/**
 * Overrides the parent implementation to install a SocketProxy in-front of the Jetty server.
 */
@Override
public JettySolrRunner createJetty(File solrHome, String dataDir,
                                   String shardList, String solrConfigOverride, String schemaOverride, Replica.Type replicaType)
    throws Exception
{
  return createProxiedJetty(solrHome, dataDir, shardList, solrConfigOverride, schemaOverride, replicaType);
}
 

public static void uploadKey(byte[] bytes, String path, MiniSolrCloudCluster cluster) throws Exception {
  JettySolrRunner jetty = cluster.getRandomJetty(random());
  try(HttpSolrClient client = (HttpSolrClient) jetty.newClient()) {
    PackageUtils.uploadKey(bytes, path, Paths.get(jetty.getCoreContainer().getSolrHome()), client);
    Object resp = Utils.executeGET(client.getHttpClient(), jetty.getBaseURLV2().toString() + "/node/files" + path + "?sync=true", null);
    System.out.println("sync resp: "+jetty.getBaseURLV2().toString() + "/node/files" + path + "?sync=true"+" ,is: "+resp);
  }
  waitForAllNodesHaveFile(cluster,path, Utils.makeMap(":files:" + path + ":name", (Predicate<Object>) Objects::nonNull),
      false);
}
 

@BeforeClass
public static void createMiniSolrCloudCluster() throws Exception {

  // 50% runs use single node/shard a FL_VALIDATORS with all validators known to work on single node
  // 50% runs use multi node/shard with FL_VALIDATORS only containing stuff that works in cloud
  final boolean singleCoreMode = random().nextBoolean();

  // (asuming multi core multi replicas shouldn't matter (assuming multi node) ...
  final int repFactor = singleCoreMode ? 1 : (usually() ? 1 : 2);
  // ... but we definitely want to ensure forwarded requests to other shards work ...
  final int numShards = singleCoreMode ? 1 : 2;
  // ... including some forwarded requests from nodes not hosting a shard
  final int numNodes = 1 + (singleCoreMode ? 0 : (numShards * repFactor));
  
  final String configName = DEBUG_LABEL + "_config-set";
  final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");
  
  configureCluster(numNodes).addConfig(configName, configDir).configure();

  CLOUD_CLIENT = cluster.getSolrClient();
  CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);

  CollectionAdminRequest.createCollection(COLLECTION_NAME, configName, numShards, repFactor)
      .withProperty("config", "solrconfig-tlog.xml")
      .withProperty("schema", "schema-psuedo-fields.xml")
      .process(CLOUD_CLIENT);

  cluster.waitForActiveCollection(COLLECTION_NAME, numShards, repFactor * numShards); 

  for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
    CLIENTS.add(getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME + "/"));
  }
}
 

public JettySolrRunner createJetty(File solrHome, String dataDir, String shardList, String solrConfigOverride, String schemaOverride, Replica.Type replicaType) throws Exception {
  // randomly test a relative solr.home path
  if (random().nextBoolean()) {
    solrHome = getRelativeSolrHomePath(solrHome);
  }

  JettyConfig jettyconfig = JettyConfig.builder()
      .setContext(context)
      .stopAtShutdown(false)
      .withServlets(getExtraServlets())
      .withFilters(getExtraRequestFilters())
      .withSSLConfig(sslConfig.buildServerSSLConfig())
      .build();

  Properties props = new Properties();
  if (solrConfigOverride != null)
    props.setProperty("solrconfig", solrConfigOverride);
  if (schemaOverride != null)
    props.setProperty("schema", schemaOverride);
  if (shardList != null)
    props.setProperty("shards", shardList);
  if (dataDir != null)
    props.setProperty("solr.data.dir", getDataDir(dataDir));
  if (replicaType != null) {
    props.setProperty("replicaType", replicaType.toString());
  } else if (random().nextBoolean()) {
    props.setProperty("replicaType", Replica.Type.NRT.toString());
  }
  props.setProperty("coreRootDirectory", solrHome.toPath().resolve("cores").toString());
  
  JettySolrRunner jetty = new JettySolrRunner(solrHome.getPath(), props, jettyconfig);

  return jetty;
}
 

private static JettySolrRunner createAndStartJetty(TestReplicationHandler.SolrInstance instance) throws Exception {
  FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new File(instance.getHomeDir(), "solr.xml"));
  Properties nodeProperties = new Properties();
  nodeProperties.setProperty("solr.data.dir", instance.getDataDir());
  JettyConfig jettyConfig = JettyConfig.builder().setContext("/solr").setPort(0).build();
  JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), nodeProperties, jettyConfig);
  jetty.start();
  return jetty;
}
 

private void testStopAndStartCoresInOneInstance() throws Exception {
  JettySolrRunner jetty = jettys.get(0);
  try (final HttpSolrClient httpSolrClient = (HttpSolrClient) jetty.newClient(15000, 60000)) {
    ThreadPoolExecutor executor = null;
    try {
      executor = new ExecutorUtil.MDCAwareThreadPoolExecutor(0, Integer.MAX_VALUE,
          5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
          new SolrNamedThreadFactory("testExecutor"));
      int cnt = 3;

      // create the cores
      createCollectionInOneInstance(httpSolrClient, jetty.getNodeName(), executor, "multiunload2", 1, cnt);
    } finally {
      if (executor != null) {
        ExecutorUtil.shutdownAndAwaitTermination(executor);
      }
    }
  }
  
  cloudJettys.get(0).jetty.stop();
  printLayout();

  cloudJettys.get(0).jetty.start();
  cloudClient.getZkStateReader().forceUpdateCollection("multiunload2");
  try {
    cloudClient.getZkStateReader().getLeaderRetry("multiunload2", "shard1", 30000);
  } catch (SolrException e) {
    printLayout();
    throw e;
  }
  
  printLayout();

}
 

/**
 * Overrides the parent implementation to install a SocketProxy in-front of the Jetty server.
 */
@Override
public JettySolrRunner createJetty(File solrHome, String dataDir,
    String shardList, String solrConfigOverride, String schemaOverride, Replica.Type replicaType)
    throws Exception
{
  return createProxiedJetty(solrHome, dataDir, shardList, solrConfigOverride, schemaOverride, replicaType);
}
 

public static void causeConnectionLoss(JettySolrRunner jetty) {
  CoreContainer cores = jetty.getCoreContainer();
  if (cores != null) {
    monkeyLog("Will cause connection loss on " + jetty.getLocalPort());
    SolrZkClient zkClient = cores.getZkController().getZkClient();
    zkClient.getSolrZooKeeper().closeCnxn();
  }
}
 

@Test
public void testHealthCheckV2Api() throws Exception {
  V2Response res = new V2Request.Builder("/node/health").build().process(cluster.getSolrClient());
  assertEquals(0, res.getStatus());
  assertEquals(CommonParams.OK, res.getResponse().get(CommonParams.STATUS));

  // add a new node for the purpose of negative testing
  JettySolrRunner newJetty = cluster.startJettySolrRunner();
  try (HttpSolrClient httpSolrClient = getHttpSolrClient(newJetty.getBaseUrl().toString())) {

    // postive check that our (new) "healthy" node works with direct http client
    assertEquals(CommonParams.OK, new V2Request.Builder("/node/health").build().process(httpSolrClient).
        getResponse().get(CommonParams.STATUS));

    // now "break" our (new) node
    newJetty.getCoreContainer().getZkController().getZkClient().close();

    // negative check of our (new) "broken" node that we deliberately put into an unhealth state
    BaseHttpSolrClient.RemoteSolrException e = expectThrows(BaseHttpSolrClient.RemoteSolrException.class, () ->
    {
      new V2Request.Builder("/node/health").build().process(httpSolrClient);
    });
    assertTrue(e.getMessage(), e.getMessage().contains("Host Unavailable"));
    assertEquals(SolrException.ErrorCode.SERVICE_UNAVAILABLE.code, e.code());
  } finally {
    newJetty.stop();
  }
}
 

private void checkInstanceDirs(JettySolrRunner jetty) throws IOException {
  CoreContainer cores = jetty.getCoreContainer();
  Collection<SolrCore> theCores = cores.getCores();
  for (SolrCore core : theCores) {
    // look for core props file
    Path instancedir = core.getInstancePath();
    assertTrue("Could not find expected core.properties file", Files.exists(instancedir.resolve("core.properties")));

    Path expected = Paths.get(jetty.getSolrHome()).toAbsolutePath().resolve(core.getName());

    assertTrue("Expected: " + expected + "\nFrom core stats: " + instancedir, Files.isSameFile(expected, instancedir));
  }
}
 

@Test
public void testOtherReplicasAreNotActive() throws Exception {
  final String collection = "collection2";
  cluster.getSolrClient().setDefaultCollection(collection);
  int numReplicas = random().nextInt(2) + 1;
  // won't do anything if leader is the only one active replica in the shard
  CollectionAdminRequest
      .createCollection(collection, "config", 1, numReplicas)
      .process(cluster.getSolrClient());
  cluster.waitForActiveCollection(collection, 1, numReplicas);

  try {
    JettySolrRunner otherReplicaJetty = null;
    if (numReplicas == 2) {
      Slice shard = getCollectionState(collection).getSlice("shard1");
      otherReplicaJetty = cluster.getReplicaJetty(getNonLeader(shard));
      if (log.isInfoEnabled()) {
        log.info("Stop jetty node : {} state:{}", otherReplicaJetty.getBaseUrl(), getCollectionState(collection));
      }
      otherReplicaJetty.stop();
      cluster.waitForJettyToStop(otherReplicaJetty);
      waitForState("Timeout waiting for replica get down", collection, (liveNodes, collectionState) -> getNonLeader(collectionState.getSlice("shard1")).getState() != Replica.State.ACTIVE);
    }

    Replica oldLeader = corruptLeader(collection, new ArrayList<>());

    if (otherReplicaJetty != null) {
      otherReplicaJetty.start();
      cluster.waitForNode(otherReplicaJetty, 30);
    }

    Replica leader = getCollectionState(collection).getSlice("shard1").getLeader();
    assertEquals(leader.getName(), oldLeader.getName());
  } finally {
    CollectionAdminRequest.deleteCollection(collection).process(cluster.getSolrClient());
  }
}