/**
 * Helper method for testing async snapshot operations. Just waits for the given snapshot to
 * complete on the server by repeatedly checking the master.
 * @param master the master running the snapshot
 * @param snapshot the snapshot to check
 * @param timeoutNanos the timeout in nano between checks to see if the snapshot is done
 */
private static void waitForSnapshotToComplete(HMaster master,
    SnapshotProtos.SnapshotDescription snapshot, long timeoutNanos) throws Exception {
  final IsSnapshotDoneRequest request =
    IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot).build();
  long start = System.nanoTime();
  while (System.nanoTime() - start < timeoutNanos) {
    try {
      IsSnapshotDoneResponse done = master.getMasterRpcServices().isSnapshotDone(null, request);
      if (done.getDone()) {
        return;
      }
    } catch (ServiceException e) {
      // ignore UnknownSnapshotException, this is possible as for AsyncAdmin, the method will
      // return immediately after sending out the request, no matter whether the master has
      // processed the request or not.
      if (!(e.getCause() instanceof UnknownSnapshotException)) {
        throw e;
      }
    }

    Thread.sleep(200);
  }
  throw new TimeoutException("Timeout waiting for snapshot " + snapshot + " to complete");
}
 

@Override
public void postCloneSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, SnapshotDescription snapshot, TableDescriptor tableDescriptor) throws IOException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("==> HBaseAtlasCoprocessor.postCloneSnapshot()");
    }

    try {
        activatePluginClassLoader();
        implMasterObserver.postCloneSnapshot(observerContext,snapshot,tableDescriptor);
    } finally {
        deactivatePluginClassLoader();
    }

    if (LOG.isDebugEnabled()) {
        LOG.debug("<== HBaseAtlasCoprocessor.postCloneSnapshot()");
    }
}
 

@Override
public void preCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor)
    throws IOException {
  User user = getActiveUser(ctx);
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)
      && hTableDescriptor.getTableName().getNameAsString()
      .equals(snapshot.getTableNameAsString())) {
    // Snapshot owner is allowed to create a table with the same name as the snapshot he took
    AuthResult result = AuthResult.allow("cloneSnapshot " + snapshot.getName(),
      "Snapshot owner check allowed", user, null, hTableDescriptor.getTableName(), null);
    AccessChecker.logResult(result);
  } else {
    accessChecker.requirePermission(user, "cloneSnapshot " + snapshot.getName(), null,
      Action.ADMIN);
  }
}
 

/**
 * Returns the list of available snapshots in the specified location
 * @param conf the {@link Configuration} to use
 * @return the list of snapshots
 */
public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
    throws IOException {
  Path rootDir = CommonFSUtils.getRootDir(conf);
  FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
  Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
  FileStatus[] snapshots = fs.listStatus(snapshotDir,
      new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
  List<SnapshotDescription> snapshotLists = new ArrayList<>(snapshots.length);
  for (FileStatus snapshotDirStat: snapshots) {
    SnapshotProtos.SnapshotDescription snapshotDesc =
        SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath());
    snapshotLists.add(ProtobufUtil.createSnapshotDesc(snapshotDesc));
  }
  return snapshotLists;
}
 

private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
    final int numRegions, final int version) throws IOException {
  TableDescriptor htd = createHtd(tableName);
  RegionData[] regions = createTable(htd, numRegions);

  SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder()
    .setTable(htd.getTableName().getNameAsString())
    .setName(snapshotName)
    .setVersion(version)
    .build();

  Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);
  FileSystem workingFs = workingDir.getFileSystem(conf);
  SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, workingFs);
  return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
}
 

private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException {
  HashSet<String> files = new HashSet<>();
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
      snapshotName, CommonFSUtils.getRootDir(conf));
  SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(
      fs, snapshotDir);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
  // For each region referenced by the snapshot
  for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
    // For each column family in this region
    for (FamilyFiles ff : rm.getFamilyFilesList()) {
      // And each store file in that family
      for (StoreFile sf : ff.getStoreFilesList()) {
        files.add(sf.getName());
      }
    }
  }
  return files;
}
 

private SnapshotProtos.SnapshotDescription getSnapshot() throws Exception {
  if (snapshot == null) {
    final TableName snapshotTableName = TableName.valueOf("testCloneSnapshot");
    long tid = System.currentTimeMillis();
    final String snapshotName = "snapshot-" + tid;

    Admin admin = UTIL.getAdmin();
    // create Table
    SnapshotTestingUtils.createTable(UTIL, snapshotTableName, getNumReplicas(), CF);
    // Load data
    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, 500, CF);
    admin.disableTable(snapshotTableName);
    // take a snapshot
    admin.snapshot(snapshotName, snapshotTableName);
    admin.enableTable(snapshotTableName);

    List<SnapshotDescription> snapshotList = admin.listSnapshots();
    snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshotList.get(0));
  }
  return snapshot;
}
 

@Test
public void testCloneSnapshot() throws Exception {
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  final TableName clonedTableName = TableName.valueOf("testCloneSnapshot2");
  final TableDescriptor htd = createTableDescriptor(clonedTableName, CF);

  // take the snapshot
  SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot();

  long procId = ProcedureTestingUtility.submitAndWait(
    procExec, new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc));
  ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
  MasterProcedureTestingUtility.validateTableIsEnabled(
    UTIL.getHBaseCluster().getMaster(),
    clonedTableName);
}
 

@Test
public void testCloneSnapshotToSameTable() throws Exception {
  // take the snapshot
  SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot();

  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  final TableName clonedTableName = TableName.valueOf(snapshotDesc.getTable());
  final TableDescriptor htd = createTableDescriptor(clonedTableName, CF);

  long procId = ProcedureTestingUtility.submitAndWait(
    procExec, new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc));
  Procedure<?> result = procExec.getResult(procId);
  assertTrue(result.isFailed());
  LOG.debug("Clone snapshot failed with exception: " + result.getException());
  assertTrue(
    ProcedureTestingUtility.getExceptionCause(result) instanceof TableExistsException);
}
 

private void upsertAndSnapshot(String tableName, boolean shouldSplit) throws Exception {
  upsertData(tableName);

  TableName hbaseTableName = TableName.valueOf(tableName);
  Connection conn = DriverManager.getConnection(getUrl());
  Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();

  if (shouldSplit) {
    splitTableSync(admin, hbaseTableName, "BBBB".getBytes(), 2);
  }

  admin.snapshot(SNAPSHOT_NAME, hbaseTableName);

  List<SnapshotDescription> snapshots = admin.listSnapshots();
  Assert.assertEquals(tableName, snapshots.get(0).getTable());

  // Capture the snapshot timestamp to use as SCN while reading the table later
  // Assigning the timestamp value here will make tests less flaky
  timestamp = System.currentTimeMillis();

  // upsert data after snapshot
  PreparedStatement stmt = conn.prepareStatement(String.format(UPSERT, tableName));
  upsertData(stmt, "DDDD", "SNFB", 45);
  conn.commit();
}
 

/**
 * Make sure that there is only one snapshot returned from the master and its
 * name and table match the passed in parameters.
 */
public static List<SnapshotDescription> assertExistsMatchingSnapshot(
    Admin admin, String snapshotName, TableName tableName)
    throws IOException {
  // list the snapshot
  List<SnapshotDescription> snapshots = admin.listSnapshots();

  List<SnapshotDescription> returnedSnapshots = new ArrayList<>();
  for (SnapshotDescription sd : snapshots) {
    if (snapshotName.equals(sd.getName()) && tableName.equals(sd.getTableName())) {
      returnedSnapshots.add(sd);
    }
  }

  Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
  return returnedSnapshots;
}
 

@Override
public void postRestoreSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, SnapshotDescription snapshot, TableDescriptor tableDescriptor) throws IOException {
    LOG.info("==> HBaseAtlasCoprocessor.postRestoreSnapshot()");

    hbaseAtlasHook.sendHBaseTableOperation(tableDescriptor, snapshot.getTableName(), HBaseAtlasHook.OPERATION.ALTER_TABLE, observerContext);

    if (LOG.isDebugEnabled()) {
        LOG.debug("<== HBaseAtlasCoprocessor.postRestoreSnapshot()");
    }
}
 

protected static boolean snapshotExists(Admin admin, String snapshotName) throws IOException {
  List<SnapshotDescription> list = admin.listSnapshots();
  for (SnapshotDescription desc : list) {
    if (desc.getName().equals(snapshotName)) {
      return true;
    }
  }
  return false;
}
 

@Override
public void preSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor)
    throws IOException {
  if (failures.contains(Failure.PRE_SNAPSHOT_FAILURE)) {
    throw new IOException("preSnapshot");
  }
}
 

@Override
public void preCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor)
    throws IOException {
  if (cloneCount != null) {
    cloneCount.incrementAndGet();
  }
}
 

@Override
public void postDeleteSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
    SnapshotDescription snapshot) throws IOException {
  if (failures.contains(Failure.POST_DELETE_SNAPSHOT_FAILURE)) {
    throw new IOException("postDeleteSnapshot");
  }
}
 

private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
    final int numRegions, final int version, final long ttl) throws IOException {
  TableDescriptor htd = createHtd(tableName);
  RegionData[] regions = createTable(htd, numRegions);
  SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder()
      .setTable(htd.getTableName().getNameAsString())
      .setName(snapshotName)
      .setVersion(version)
      .setCreationTime(EnvironmentEdgeManager.currentTime())
      .setTtl(ttl)
      .build();
  Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);
  SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
  return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
}
 

@Override
public void preListSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot) throws IOException {
  User user = getActiveUser(ctx);
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)) {
    // list it, if user is the owner of snapshot
    AuthResult result = AuthResult.allow("listSnapshot " + snapshot.getName(),
        "Snapshot owner check allowed", user, null, null, null);
    AccessChecker.logResult(result);
  } else {
    accessChecker.requirePermission(user, "listSnapshot " + snapshot.getName(), null,
      Action.ADMIN);
  }
}
 

@Override
public void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot) throws IOException {
  User user = getActiveUser(ctx);
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)) {
    // Snapshot owner is allowed to delete the snapshot
    AuthResult result = AuthResult.allow("deleteSnapshot " + snapshot.getName(),
        "Snapshot owner check allowed", user, null, null, null);
    AccessChecker.logResult(result);
  } else {
    accessChecker.requirePermission(user, "deleteSnapshot " + snapshot.getName(), null,
      Action.ADMIN);
  }
}
 

@Override
public void postCompletedSnapshotAction(ObserverContext<MasterCoprocessorEnvironment> c,
    SnapshotDescription snapshot, TableDescriptor tableDescriptor) throws IOException {
  if (needHandleTableHdfsAcl(tableDescriptor, "snapshot " + snapshot.getName())) {
    // Add HDFS acls of users with table read permission to snapshot files
    hdfsAclHelper.snapshotAcl(snapshot);
  }
}
 

/**
 * Take a snapshot of the specified table and verify the given families.
 * Note that this will leave the table disabled in the case of an offline snapshot.
 */
public static void createSnapshotAndValidate(Admin admin,
    TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
    String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
      throws Exception {
  if (!onlineSnapshot) {
    try {
      LOG.info("prepping for offline snapshot.");
      admin.disableTable(tableName);
    } catch (TableNotEnabledException tne) {
      LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
          "already disabled.");
    }
  }
  LOG.info("taking snapshot.");
  admin.snapshot(snapshotNameString, tableName);

  LOG.info("Confirming snapshot exists.");
  List<SnapshotDescription> snapshots =
      SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName);
  if (snapshots == null || snapshots.size() != 1) {
    Assert.fail("Incorrect number of snapshots for table " + tableName);
  }

  LOG.info("validating snapshot.");
  SnapshotTestingUtils.confirmSnapshotValid(
    ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames,
    emptyFamilyNames, rootDir, admin, fs);
}
 

public void preSnapshot(final SnapshotDescription snapshot,
    final TableDescriptor hTableDescriptor) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.preSnapshot(this, snapshot, hTableDescriptor);
    }
  });
}
 

public void postSnapshot(final SnapshotDescription snapshot,
    final TableDescriptor hTableDescriptor) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.postSnapshot(this, snapshot, hTableDescriptor);
    }
  });
}
 

public void postCompletedSnapshotAction(SnapshotDescription snapshot,
    TableDescriptor hTableDescriptor) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.postCompletedSnapshotAction(this, snapshot, hTableDescriptor);
    }
  });
}
 

public void preListSnapshot(final SnapshotDescription snapshot) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.preListSnapshot(this, snapshot);
    }
  });
}
 

/**
 * Test snapshotting a table that is online without flushing
 */
@Test
public void testSkipFlushTableSnapshot() throws Exception {
  // make sure we don't fail on listing snapshots
  SnapshotTestingUtils.assertNoSnapshots(admin);

  // put some stuff in the table
  Table table = UTIL.getConnection().getTable(TABLE_NAME);
  UTIL.loadTable(table, TEST_FAM);
  UTIL.flush(TABLE_NAME);

  LOG.debug("FS state before snapshot:");
  UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);

  // take a snapshot of the enabled table
  String snapshotString = "skipFlushTableSnapshot";
  String snapshot = snapshotString;
  admin.snapshot(snapshotString, TABLE_NAME, SnapshotType.SKIPFLUSH);
  LOG.debug("Snapshot completed.");

  // make sure we have the snapshot
  List<SnapshotDescription> snapshots =
      SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);

  // make sure its a valid snapshot
  LOG.debug("FS state after snapshot:");
  UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);

  SnapshotTestingUtils.confirmSnapshotValid(UTIL,
    ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM);

  admin.deleteSnapshot(snapshot);
  snapshots = admin.listSnapshots();
  SnapshotTestingUtils.assertNoSnapshots(admin);
}
 

public void preCloneSnapshot(final SnapshotDescription snapshot,
    final TableDescriptor hTableDescriptor) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.preCloneSnapshot(this, snapshot, hTableDescriptor);
    }
  });
}
 

public void preRestoreSnapshot(final SnapshotDescription snapshot,
    final TableDescriptor hTableDescriptor) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.preRestoreSnapshot(this, snapshot, hTableDescriptor);
    }
  });
}
 

public void postRestoreSnapshot(final SnapshotDescription snapshot,
    final TableDescriptor hTableDescriptor) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.postRestoreSnapshot(this, snapshot, hTableDescriptor);
    }
  });
}
 

public void postDeleteSnapshot(final SnapshotDescription snapshot) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.postDeleteSnapshot(this, snapshot);
    }
  });
}