当前位置：源码 > Java源码 >

Java源码示例：org.apache.kylin.job.exception.JobException

示例1

public static double getTotalMapInputMB(Job job)
        throws ClassNotFoundException, IOException, InterruptedException, JobException {
    if (job == null) {
        throw new JobException("Job is null");
    }

    long mapInputBytes = 0;
    InputFormat<?, ?> input = ReflectionUtils.newInstance(job.getInputFormatClass(), job.getConfiguration());
    for (InputSplit split : input.getSplits(job)) {
        mapInputBytes += split.getLength();
    }
    
    // 0 input bytes is possible when the segment range hits no partition on a partitioned hive table (KYLIN-2470) 
    if (mapInputBytes == 0) {
        logger.warn("Map input splits are 0 bytes, something is wrong?");
    }
    
    double totalMapInputMB = (double) mapInputBytes / 1024 / 1024;
    return totalMapInputMB;
}

示例2

@Test
public void testBasics() throws JobException, IOException, SQLException {
    Assert.assertNotNull(queryService.getConfig());
    Assert.assertNotNull(queryService.getConfig());
    Assert.assertNotNull(queryService.getDataModelManager());
    Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));

    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);
    //
    //        queryService.saveQuery("test", "test", "select * from test_table", "test");
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 1);
    //
    //        queryService.removeQuery(queryService.getQueries("ADMIN").get(0).getProperty("id"));
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);

    SQLRequest request = new SQLRequest();
    request.setSql("select * from test_table");
    request.setAcceptPartial(true);
    QueryContext queryContext = QueryContextFacade.current();
    SQLResponse response = new SQLResponse();
    response.setHitExceptionCache(true);
    queryService.logQuery(queryContext.getQueryId(), request, response);
}

示例3

@Test
public void testSuccessModelUpdate() throws IOException, JobException {
    Serializer<DataModelDesc> serializer = modelService.getDataModelManager().getDataModelSerializer();
    
    List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_inner_join_model", "default", true);
    Assert.assertTrue(dataModelDescs.size() == 1);

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    serializer.serialize(dataModelDescs.get(0), new DataOutputStream(baos));
    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    DataModelDesc deserialize = serializer.deserialize(new DataInputStream(bais));

    deserialize.setOwner("somebody");
    DataModelDesc dataModelDesc = modelService.updateModelAndDesc("default", deserialize);
    Assert.assertTrue(dataModelDesc.getOwner().equals("somebody"));
}

示例4

public static double getTotalMapInputMB(Job job)
        throws ClassNotFoundException, IOException, InterruptedException, JobException {
    if (job == null) {
        throw new JobException("Job is null");
    }

    long mapInputBytes = 0;
    InputFormat<?, ?> input = ReflectionUtils.newInstance(job.getInputFormatClass(), job.getConfiguration());
    for (InputSplit split : input.getSplits(job)) {
        mapInputBytes += split.getLength();
    }
    
    // 0 input bytes is possible when the segment range hits no partition on a partitioned hive table (KYLIN-2470) 
    if (mapInputBytes == 0) {
        logger.warn("Map input splits are 0 bytes, something is wrong?");
    }
    
    double totalMapInputMB = (double) mapInputBytes / 1024 / 1024;
    return totalMapInputMB;
}

示例5

@Test
public void testBasics() throws JobException, IOException, SQLException {
    Assert.assertNotNull(queryService.getConfig());
    Assert.assertNotNull(queryService.getConfig());
    Assert.assertNotNull(queryService.getDataModelManager());
    Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));

    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);
    //
    //        queryService.saveQuery("test", "test", "select * from test_table", "test");
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 1);
    //
    //        queryService.removeQuery(queryService.getQueries("ADMIN").get(0).getProperty("id"));
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);

    SQLRequest request = new SQLRequest();
    request.setSql("select * from test_table");
    request.setAcceptPartial(true);
    QueryContext queryContext = QueryContextFacade.current();
    SQLResponse response = new SQLResponse();
    response.setHitExceptionCache(true);
    queryService.logQuery(queryContext.getQueryId(), request, response);
}

示例6

@Test
public void testSuccessModelUpdate() throws IOException, JobException {
    Serializer<DataModelDesc> serializer = modelService.getDataModelManager().getDataModelSerializer();
    
    List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_inner_join_model", "default", true);
    Assert.assertTrue(dataModelDescs.size() == 1);

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    serializer.serialize(dataModelDescs.get(0), new DataOutputStream(baos));
    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    DataModelDesc deserialize = serializer.deserialize(new DataInputStream(bais));

    deserialize.setOwner("somebody");
    DataModelDesc dataModelDesc = modelService.updateModelAndDesc("default", deserialize);
    Assert.assertTrue(dataModelDesc.getOwner().equals("somebody"));
}

示例7

protected double getTotalMapInputMB() throws ClassNotFoundException, IOException, InterruptedException, JobException {
    if (job == null) {
        throw new JobException("Job is null");
    }

    long mapInputBytes = 0;
    InputFormat<?, ?> input = ReflectionUtils.newInstance(job.getInputFormatClass(), job.getConfiguration());
    for (InputSplit split : input.getSplits(job)) {
        mapInputBytes += split.getLength();
    }
    if (mapInputBytes == 0) {
        throw new IllegalArgumentException("Map input splits are 0 bytes, something is wrong!");
    }
    double totalMapInputMB = (double) mapInputBytes / 1024 / 1024;
    return totalMapInputMB;
}

示例8

/**
 * Stop all jobs belonging to this cube and clean out all segments
 *
 * @param cube
 * @return
 * @throws IOException
 * @throws CubeIntegrityException
 * @throws JobException
 */
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
@Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
public CubeInstance purgeCube(CubeInstance cube) throws IOException, JobException {
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();
    if (null != ostatus && !RealizationStatusEnum.DISABLED.equals(ostatus)) {
        throw new InternalErrorException("Only disabled cube can be purged, status of " + cubeName + " is " + ostatus);
    }

    try {
        this.releaseAllSegments(cube);
        return cube;
    } catch (IOException e) {
        throw e;
    }

}

示例9

/**
 * Update a cube status from ready to disabled.
 *
 * @return
 * @throws CubeIntegrityException
 * @throws IOException
 * @throws JobException
 */
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
@Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
public CubeInstance disableCube(CubeInstance cube) throws IOException, JobException {
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();
    if (null != ostatus && !RealizationStatusEnum.READY.equals(ostatus)) {
        throw new InternalErrorException("Only ready cube can be disabled, status of " + cubeName + " is " + ostatus);
    }

    cube.setStatus(RealizationStatusEnum.DISABLED);

    try {
        return getCubeManager().updateCube(cube);
    } catch (IOException e) {
        cube.setStatus(ostatus);
        throw e;
    }
}

示例10

public List<JobInstance> listAllJobs(final String cubeName, final String projectName, final List<JobStatusEnum> statusList, final Integer limitValue, final Integer offsetValue) throws IOException, JobException {
    Integer limit = (null == limitValue) ? 30 : limitValue;
    Integer offset = (null == offsetValue) ? 0 : offsetValue;
    List<JobInstance> jobs = listAllJobs(cubeName, projectName, statusList);
    Collections.sort(jobs);

    if (jobs.size() <= offset) {
        return Collections.emptyList();
    }

    if ((jobs.size() - offset) < limit) {
        return jobs.subList(offset, jobs.size());
    }

    return jobs.subList(offset, offset + limit);
}

示例11

@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#job, 'ADMINISTRATION') or hasPermission(#job, 'OPERATION') or hasPermission(#job, 'MANAGEMENT')")
public JobInstance cancelJob(String jobId) throws IOException, JobException {
    //        CubeInstance cube = this.getCubeManager().getCube(job.getRelatedCube());
    //        for (BuildCubeJob cubeJob: listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING))) {
    //            getExecutableManager().stopJob(cubeJob.getId());
    //        }
    final JobInstance jobInstance = getJobInstance(jobId);
    final String segmentId = jobInstance.getRelatedSegment();
    CubeInstance cubeInstance = getCubeManager().getCube(jobInstance.getRelatedCube());
    final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
    if (segment.getStatus() == SegmentStatusEnum.NEW) {
        cubeInstance.getSegments().remove(segment);
        getCubeManager().updateCube(cubeInstance);
    }
    getExecutableManager().discardJob(jobId);
    return jobInstance;
}

示例12

@Test
public void testBasics() throws JsonProcessingException, JobException, UnknownHostException {
    Assert.assertNotNull(cubeService.getConfig());
    Assert.assertNotNull(cubeService.getKylinConfig());
    Assert.assertNotNull(cubeService.getMetadataManager());
    Assert.assertNotNull(cubeService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));

    Assert.assertTrue(CubeService.getCubeDescNameFromCube("testCube").equals("testCube_desc"));
    Assert.assertTrue(CubeService.getCubeNameFromDesc("testCube_desc").equals("testCube"));

    List<CubeInstance> cubes = cubeService.getCubes(null, null, null, null);
    Assert.assertNotNull(cubes);
    CubeInstance cube = cubes.get(0);
    cubeService.isCubeDescEditable(cube.getDescriptor());
    cubeService.isCubeEditable(cube);

    cubes = cubeService.getCubes(null, null, 1, 0);
    Assert.assertTrue(cubes.size() == 1);
}

示例13

@Test
public void testBasics() throws JobException, IOException, SQLException {
    Assert.assertNotNull(queryService.getConfig());
    Assert.assertNotNull(queryService.getKylinConfig());
    Assert.assertNotNull(queryService.getMetadataManager());
    Assert.assertNotNull(queryService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));

    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);
    //
    //        queryService.saveQuery("test", "test", "select * from test_table", "test");
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 1);
    //
    //        queryService.removeQuery(queryService.getQueries("ADMIN").get(0).getProperty("id"));
    //        Assert.assertTrue(queryService.getQueries("ADMIN").size() == 0);

    SQLRequest request = new SQLRequest();
    request.setSql("select * from test_table");
    request.setAcceptPartial(true);
    SQLResponse response = new SQLResponse();
    response.setHitCache(true);
    queryService.logQuery(request, response, new Date(), new Date());
}

示例14

private void run(String cubeName, long startDate, long endDate, String buildType) throws IOException, JobException {
    CubeInstance cube = cubeManager.getCube(cubeName);
    Preconditions.checkArgument(cube != null, "Cube named " + cubeName + " does not exist!!!");
    CubeBuildTypeEnum buildTypeEnum = CubeBuildTypeEnum.valueOf(buildType);
    Preconditions.checkArgument(buildTypeEnum != null, "Build type named " + buildType + " does not exist!!!");
    submitJob(cube, new TSRange(startDate, endDate), buildTypeEnum, false, "SYSTEM");
}

示例15

@Test
public void testSchedulerLock() throws Exception {
    if (!lock(jobLock1, jobId1)) {
        throw new JobException("fail to get the lock");
    }
    DefaultChainedExecutable job = new DefaultChainedExecutable();
    job.setId(jobId1);
    AbstractExecutable task1 = new SucceedTestExecutable();
    AbstractExecutable task2 = new SucceedTestExecutable();
    AbstractExecutable task3 = new SucceedTestExecutable();
    job.addTask(task1);
    job.addTask(task2);
    job.addTask(task3);
    execMgr.addJob(job);

    Assert.assertEquals(serverName1, getServerName(jobId1));

    waitForJobFinish(job.getId());

    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task2.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task3.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(job.getId()).getState());
    
    Thread.sleep(5000);

    Assert.assertEquals(null, getServerName(jobId1));
}

示例16

@Test
public void testSchedulerTakeOver() throws Exception {
    if (!lock(jobLock1, jobId2)) {
        throw new JobException("fail to get the lock");
    }

    DefaultChainedExecutable job = new DefaultChainedExecutable();
    job.setId(jobId2);
    AbstractExecutable task1 = new SucceedTestExecutable();
    AbstractExecutable task2 = new SucceedTestExecutable();
    AbstractExecutable task3 = new SucceedTestExecutable();
    job.addTask(task1);
    job.addTask(task2);
    job.addTask(task3);
    execMgr.addJob(job);

    waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500);

    scheduler1.shutdown();
    scheduler1 = null;

    waitForJobFinish(job.getId());

    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task2.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task3.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(job.getId()).getState());
}

示例17

protected int getMapInputSplitCount()
        throws ClassNotFoundException, JobException, IOException, InterruptedException {
    if (job == null) {
        throw new JobException("Job is null");
    }
    InputFormat<?, ?> input = ReflectionUtils.newInstance(job.getInputFormatClass(), job.getConfiguration());
    return input.getSplits(job).size();
}

示例18

public void kill() throws JobException {
    if (job != null) {
        try {
            job.killJob();
        } catch (IOException e) {
            throw new JobException(e);
        }
    }
}

示例19

public Map<String, String> getInfo() throws JobException {
    if (job != null) {
        Map<String, String> status = new HashMap<String, String>();
        if (null != job.getJobID()) {
            status.put(JobInstance.MR_JOB_ID, job.getJobID().toString());
        }
        if (null != job.getTrackingURL()) {
            status.put(JobInstance.YARN_APP_URL, job.getTrackingURL().toString());
        }

        return status;
    } else {
        throw new JobException("Job is null");
    }
}

示例20

public Counters getCounters() throws JobException {
    if (job != null) {
        try {
            return job.getCounters();
        } catch (IOException e) {
            throw new JobException(e);
        }
    } else {
        throw new JobException("Job is null");
    }
}

示例21

@Test
public void testBasics() throws JsonProcessingException, JobException, UnknownHostException, SQLException {
    Assert.assertNotNull(cubeService.getConfig());
    Assert.assertNotNull(cubeService.getConfig());
    Assert.assertNotNull(cubeService.getDataModelManager());
    Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));

    List<CubeInstance> cubes = cubeService.listAllCubes(null, null, null, true);
    Assert.assertNotNull(cubes);
    CubeInstance cube = cubes.get(0);
}

示例22

@Test
public void testBasics() throws JobException, IOException, SQLException {
    Assert.assertNotNull(jobService.getConfig());
    Assert.assertNotNull(jobService.getConfig());
    Assert.assertNotNull(jobService.getDataModelManager());
    Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));
    Assert.assertNull(jobService.getJobInstance("job_not_exist"));
    Assert.assertNotNull(jobService.searchJobs(null, null, null, 0, 0, JobTimeFilterEnum.ALL, JobService.JobSearchMode.ALL));
}

示例23

private void run(String cubeName, long startDate, long endDate, String buildType) throws IOException, JobException {
    CubeInstance cube = cubeManager.getCube(cubeName);
    Preconditions.checkArgument(cube != null, "Cube named " + cubeName + " does not exist!!!");
    CubeBuildTypeEnum buildTypeEnum = CubeBuildTypeEnum.valueOf(buildType);
    Preconditions.checkArgument(buildTypeEnum != null, "Build type named " + buildType + " does not exist!!!");
    submitJob(cube, new TSRange(startDate, endDate), buildTypeEnum, false, "SYSTEM");
}

示例24

@Test
public void testSchedulerLock() throws Exception {
    if (!lock(jobLock1, jobId1)) {
        throw new JobException("fail to get the lock");
    }
    DefaultChainedExecutable job = new DefaultChainedExecutable();
    job.setId(jobId1);
    AbstractExecutable task1 = new SucceedTestExecutable();
    AbstractExecutable task2 = new SucceedTestExecutable();
    AbstractExecutable task3 = new SucceedTestExecutable();
    job.addTask(task1);
    job.addTask(task2);
    job.addTask(task3);
    execMgr.addJob(job);

    Assert.assertEquals(serverName1, getServerName(jobId1));

    waitForJobFinish(job.getId());

    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task2.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task3.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(job.getId()).getState());
    
    Thread.sleep(5000);

    Assert.assertEquals(null, getServerName(jobId1));
}

示例25

@Test
public void testSchedulerTakeOver() throws Exception {
    if (!lock(jobLock1, jobId2)) {
        throw new JobException("fail to get the lock");
    }

    DefaultChainedExecutable job = new DefaultChainedExecutable();
    job.setId(jobId2);
    AbstractExecutable task1 = new SucceedTestExecutable();
    AbstractExecutable task2 = new SucceedTestExecutable();
    AbstractExecutable task3 = new SucceedTestExecutable();
    job.addTask(task1);
    job.addTask(task2);
    job.addTask(task3);
    execMgr.addJob(job);

    waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500);

    scheduler1.shutdown();
    scheduler1 = null;

    waitForJobFinish(job.getId());

    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task2.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task3.getId()).getState());
    Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(job.getId()).getState());
}

示例26

protected int getMapInputSplitCount()
        throws ClassNotFoundException, JobException, IOException, InterruptedException {
    if (job == null) {
        throw new JobException("Job is null");
    }
    InputFormat<?, ?> input = ReflectionUtils.newInstance(job.getInputFormatClass(), job.getConfiguration());
    return input.getSplits(job).size();
}

示例27

public void kill() throws JobException {
    if (job != null) {
        try {
            job.killJob();
        } catch (IOException e) {
            throw new JobException(e);
        }
    }
}

示例28

public Map<String, String> getInfo() throws JobException {
    if (job != null) {
        Map<String, String> status = new HashMap<String, String>();
        if (null != job.getJobID()) {
            status.put(JobInstance.MR_JOB_ID, job.getJobID().toString());
        }
        if (null != job.getTrackingURL()) {
            status.put(JobInstance.YARN_APP_URL, job.getTrackingURL().toString());
        }

        return status;
    } else {
        throw new JobException("Job is null");
    }
}

示例29

public Counters getCounters() throws JobException {
    if (job != null) {
        try {
            return job.getCounters();
        } catch (IOException e) {
            throw new JobException(e);
        }
    } else {
        throw new JobException("Job is null");
    }
}

示例30

@Test
public void testBasics() throws JsonProcessingException, JobException, UnknownHostException, SQLException {
    Assert.assertNotNull(cubeService.getConfig());
    Assert.assertNotNull(cubeService.getConfig());
    Assert.assertNotNull(cubeService.getDataModelManager());
    Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));

    List<CubeInstance> cubes = cubeService.listAllCubes(null, null, null, true);
    Assert.assertNotNull(cubes);
    CubeInstance cube = cubes.get(0);
}

Java源码示例：org.apache.kylin.job.exception.JobException

微信关注