当前位置：源码 > Java源码 >

Java源码示例：org.apache.hadoop.io.ReadaheadPool

示例1

public FadvisedFileRegion(RandomAccessFile file, long position, long count,
    boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
    String identifier, int shuffleBufferSize, 
    boolean shuffleTransferToAllowed) throws IOException {
  super(file.getChannel(), position, count);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
  this.fileChannel = file.getChannel();
  this.count = count;
  this.position = position;
  this.shuffleBufferSize = shuffleBufferSize;
  this.shuffleTransferToAllowed = shuffleTransferToAllowed;
}

示例2

public FadvisedFileRegion(RandomAccessFile file, long position, long count,
    boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
    String identifier, int shuffleBufferSize, 
    boolean shuffleTransferToAllowed) throws IOException {
  super(file.getChannel(), position, count);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
  this.fileChannel = file.getChannel();
  this.count = count;
  this.position = position;
  this.shuffleBufferSize = shuffleBufferSize;
  this.shuffleTransferToAllowed = shuffleTransferToAllowed;
}

示例3

public FadvisedFileRegion(RandomAccessFile file, long position, long count,
                          boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
                          String identifier, int shuffleBufferSize,
                          boolean shuffleTransferToAllowed) throws IOException {
  super(file.getChannel(), position, count);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
  this.fileChannel = file.getChannel();
  this.count = count;
  this.position = position;
  this.shuffleBufferSize = shuffleBufferSize;
  this.shuffleTransferToAllowed = shuffleTransferToAllowed;
}

示例4

public FadvisedFileRegion(RandomAccessFile file, long position, long count,
                          boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
                          String identifier, int shuffleBufferSize,
                          boolean shuffleTransferToAllowed) throws IOException {
  super(file.getChannel(), position, count);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
  this.fileChannel = file.getChannel();
  this.count = count;
  this.position = position;
  this.shuffleBufferSize = shuffleBufferSize;
  this.shuffleTransferToAllowed = shuffleTransferToAllowed;
}

示例5

public FadvisedFileRegion(RandomAccessFile file, long position, long count,
                          boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
                          String identifier, int shuffleBufferSize,
                          boolean shuffleTransferToAllowed) throws IOException {
  super(file.getChannel(), position, count);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
  this.fileChannel = file.getChannel();
  this.count = count;
  this.position = position;
  this.shuffleBufferSize = shuffleBufferSize;
  this.shuffleTransferToAllowed = shuffleTransferToAllowed;
}

示例6

public FadvisedChunkedFile(RandomAccessFile file, long position, long count,
    int chunkSize, boolean manageOsCache, int readaheadLength,
    ReadaheadPool readaheadPool, String identifier) throws IOException {
  super(file, position, count, chunkSize);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
}

示例7

public FadvisedChunkedFile(RandomAccessFile file, long position, long count,
    int chunkSize, boolean manageOsCache, int readaheadLength,
    ReadaheadPool readaheadPool, String identifier) throws IOException {
  super(file, position, count, chunkSize);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
}

示例8

public FadvisedChunkedFile(RandomAccessFile file, long position, long count,
                           int chunkSize, boolean manageOsCache, int readaheadLength,
                           ReadaheadPool readaheadPool, String identifier) throws IOException {
  super(file, position, count, chunkSize);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
}

示例9

public FadvisedChunkedFile(RandomAccessFile file, long position, long count,
                           int chunkSize, boolean manageOsCache, int readaheadLength,
                           ReadaheadPool readaheadPool, String identifier) throws IOException {
  super(file, position, count, chunkSize);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
}

示例10

public FadvisedFileRegion(RandomAccessFile file, long position, long count,
    boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
    String identifier) throws IOException {
  super(file.getChannel(), position, count);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
}

示例11

public FadvisedChunkedFile(RandomAccessFile file, long position, long count,
    int chunkSize, boolean manageOsCache, int readaheadLength,
    ReadaheadPool readaheadPool, String identifier) throws IOException {
  super(file, position, count, chunkSize);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
}

示例12

public FadvisedChunkedFile(RandomAccessFile file, long position, long count,
                           int chunkSize, boolean manageOsCache, int readaheadLength,
                           ReadaheadPool readaheadPool, String identifier) throws IOException {
  super(file, position, count, chunkSize);
  this.manageOsCache = manageOsCache;
  this.readaheadLength = readaheadLength;
  this.readaheadPool = readaheadPool;
  this.fd = file.getFD();
  this.identifier = identifier;
}

示例13

/**
 * This method starts the data node with the specified conf.
 * 
 * @param conf - the configuration
 *  if conf's CONFIG_PROPERTY_SIMULATED property is set
 *  then a simulated storage based data node is created.
 * 
 * @param dataDirs - only for a non-simulated storage data node
 * @throws IOException
 */
void startDataNode(Configuration conf, 
                   List<StorageLocation> dataDirs,
                   SecureResources resources
                   ) throws IOException {

  // settings global for all BPs in the Data Node
  this.secureResources = resources;
  synchronized (this) {
    this.dataDirs = dataDirs;
  }
  this.conf = conf;
  this.dnConf = new DNConf(conf);
  checkSecureConfig(dnConf, conf, resources);

  this.spanReceiverHost =
    SpanReceiverHost.get(conf, DFSConfigKeys.DFS_SERVER_HTRACE_PREFIX);

  if (dnConf.maxLockedMemory > 0) {
    if (!NativeIO.POSIX.getCacheManipulator().verifyCanMlock()) {
      throw new RuntimeException(String.format(
          "Cannot start datanode because the configured max locked memory" +
          " size (%s) is greater than zero and native code is not available.",
          DFS_DATANODE_MAX_LOCKED_MEMORY_KEY));
    }
    if (Path.WINDOWS) {
      NativeIO.Windows.extendWorkingSetSize(dnConf.maxLockedMemory);
    } else {
      long ulimit = NativeIO.POSIX.getCacheManipulator().getMemlockLimit();
      if (dnConf.maxLockedMemory > ulimit) {
        throw new RuntimeException(String.format(
          "Cannot start datanode because the configured max locked memory" +
          " size (%s) of %d bytes is more than the datanode's available" +
          " RLIMIT_MEMLOCK ulimit of %d bytes.",
          DFS_DATANODE_MAX_LOCKED_MEMORY_KEY,
          dnConf.maxLockedMemory,
          ulimit));
      }
    }
  }
  LOG.info("Starting DataNode with maxLockedMemory = " +
      dnConf.maxLockedMemory);

  storage = new DataStorage();
  
  // global DN settings
  registerMXBean();
  initDataXceiver(conf);
  startInfoServer(conf);
  pauseMonitor = new JvmPauseMonitor(conf);
  pauseMonitor.start();

  // BlockPoolTokenSecretManager is required to create ipc server.
  this.blockPoolTokenSecretManager = new BlockPoolTokenSecretManager();

  // Login is done by now. Set the DN user name.
  dnUserName = UserGroupInformation.getCurrentUser().getShortUserName();
  LOG.info("dnUserName = " + dnUserName);
  LOG.info("supergroup = " + supergroup);
  initIpcServer(conf);

  metrics = DataNodeMetrics.create(conf, getDisplayName());
  metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
  
  blockPoolManager = new BlockPoolManager(this);
  blockPoolManager.refreshNamenodes(conf);

  // Create the ReadaheadPool from the DataNode context so we can
  // exit without having to explicitly shutdown its thread pool.
  readaheadPool = ReadaheadPool.getInstance();
  saslClient = new SaslDataTransferClient(dnConf.conf, 
      dnConf.saslPropsResolver, dnConf.trustedChannelResolver);
  saslServer = new SaslDataTransferServer(dnConf, blockPoolTokenSecretManager);
}

示例14

/**
 * This method starts the data node with the specified conf.
 * 
 * @param conf - the configuration
 *  if conf's CONFIG_PROPERTY_SIMULATED property is set
 *  then a simulated storage based data node is created.
 * 
 * @param dataDirs - only for a non-simulated storage data node
 * @throws IOException
 */
void startDataNode(Configuration conf, 
                   List<StorageLocation> dataDirs,
                   SecureResources resources
                   ) throws IOException {

  // settings global for all BPs in the Data Node
  this.secureResources = resources;
  synchronized (this) {
    this.dataDirs = dataDirs;
  }
  this.conf = conf;
  this.dnConf = new DNConf(conf);
  checkSecureConfig(dnConf, conf, resources);

  this.spanReceiverHost =
    SpanReceiverHost.get(conf, DFSConfigKeys.DFS_SERVER_HTRACE_PREFIX);

  if (dnConf.maxLockedMemory > 0) {
    if (!NativeIO.POSIX.getCacheManipulator().verifyCanMlock()) {
      throw new RuntimeException(String.format(
          "Cannot start datanode because the configured max locked memory" +
          " size (%s) is greater than zero and native code is not available.",
          DFS_DATANODE_MAX_LOCKED_MEMORY_KEY));
    }
    if (Path.WINDOWS) {
      NativeIO.Windows.extendWorkingSetSize(dnConf.maxLockedMemory);
    } else {
      long ulimit = NativeIO.POSIX.getCacheManipulator().getMemlockLimit();
      if (dnConf.maxLockedMemory > ulimit) {
        throw new RuntimeException(String.format(
          "Cannot start datanode because the configured max locked memory" +
          " size (%s) of %d bytes is more than the datanode's available" +
          " RLIMIT_MEMLOCK ulimit of %d bytes.",
          DFS_DATANODE_MAX_LOCKED_MEMORY_KEY,
          dnConf.maxLockedMemory,
          ulimit));
      }
    }
  }
  LOG.info("Starting DataNode with maxLockedMemory = " +
      dnConf.maxLockedMemory);

  storage = new DataStorage();
  
  // global DN settings
  registerMXBean();
  initDataXceiver(conf);
  startInfoServer(conf);
  pauseMonitor = new JvmPauseMonitor(conf);
  pauseMonitor.start();

  // BlockPoolTokenSecretManager is required to create ipc server.
  this.blockPoolTokenSecretManager = new BlockPoolTokenSecretManager();

  // Login is done by now. Set the DN user name.
  dnUserName = UserGroupInformation.getCurrentUser().getShortUserName();
  LOG.info("dnUserName = " + dnUserName);
  LOG.info("supergroup = " + supergroup);
  initIpcServer(conf);

  metrics = DataNodeMetrics.create(conf, getDisplayName());
  metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
  
  blockPoolManager = new BlockPoolManager(this);
  blockPoolManager.refreshNamenodes(conf);

  // Create the ReadaheadPool from the DataNode context so we can
  // exit without having to explicitly shutdown its thread pool.
  readaheadPool = ReadaheadPool.getInstance();
  saslClient = new SaslDataTransferClient(dnConf.conf, 
      dnConf.saslPropsResolver, dnConf.trustedChannelResolver);
  saslServer = new SaslDataTransferServer(dnConf, blockPoolTokenSecretManager);
}

示例15

public FadvisedFileRegion(RandomAccessFile file, long position, long count,
                          boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
                          String identifier) throws IOException {
  this(file, position, count, manageOsCache, readaheadLength, readaheadPool,
      identifier, DEFAULT_SHUFFLE_BUFFER_SIZE, true);
}

示例16

public FadvisedFileRegion(RandomAccessFile file, long position, long count,
                          boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
                          String identifier) throws IOException {
  this(file, position, count, manageOsCache, readaheadLength, readaheadPool,
      identifier, DEFAULT_SHUFFLE_BUFFER_SIZE, true);
}

Java源码示例：org.apache.hadoop.io.ReadaheadPool

微信关注