Java源码示例:org.apache.flink.test.util.TestBaseUtils
示例1
@After
public void after() throws Exception {
TestBaseUtils.compareResultsByLinesInMemory(expectedTopSongs, topSongsResultPath);
ArrayList<String> list = new ArrayList<>();
TestBaseUtils.readAllResultLines(list, communitiesResultPath, new String[]{}, false);
String[] result = list.toArray(new String[list.size()]);
Arrays.sort(result);
// check that user_1 and user_2 are in the same community
Assert.assertEquals("users 1 and 2 are not in the same community",
result[0].substring(7), result[1].substring(7));
// check that user_3, user_4 and user_5 are in the same community
Assert.assertEquals("users 3 and 4 are not in the same community",
result[2].substring(7), result[3].substring(7));
Assert.assertEquals("users 4 and 5 are not in the same community",
result[3].substring(7), result[4].substring(7));
}
示例2
@Test
public void testSimpleGraph() throws Exception {
DataSet<Result<IntValue>> ji = undirectedSimpleGraph
.run(new JaccardIndex<>());
String expectedResult =
"(0,1,1,4)\n" +
"(0,2,1,4)\n" +
"(0,3,2,4)\n" +
"(1,2,2,4)\n" +
"(1,3,1,6)\n" +
"(1,4,1,3)\n" +
"(1,5,1,3)\n" +
"(2,3,1,6)\n" +
"(2,4,1,3)\n" +
"(2,5,1,3)\n" +
"(4,5,1,1)\n";
TestBaseUtils.compareResultAsText(ji.collect(), expectedResult);
}
示例3
@Test
public void testWithSimpleGraph() throws Exception {
String expectedResult =
"(0,1,((null),(2,2,0)))\n" +
"(0,2,((null),(2,2,0)))\n" +
"(2,1,((null),(3,2,1)))\n" +
"(2,3,((null),(3,2,1)))\n" +
"(3,1,((null),(4,2,2)))\n" +
"(3,4,((null),(4,2,2)))\n" +
"(5,3,((null),(1,1,0)))";
DataSet<Edge<IntValue, Tuple2<NullValue, Degrees>>> sourceDegrees = directedSimpleGraph
.run(new EdgeSourceDegrees<>());
TestBaseUtils.compareResultAsText(sourceDegrees.collect(), expectedResult);
}
示例4
@Test
public void testSourceConfig() throws Exception {
// vector reader not available for 1.x and we're not testing orc for 2.0.x
Assume.assumeTrue(HiveVersionTestUtil.HIVE_210_OR_LATER);
Map<String, String> env = System.getenv();
hiveShell.execute("create database db1");
try {
hiveShell.execute("create table db1.src (x int,y string) stored as orc");
hiveShell.execute("insert into db1.src values (1,'a'),(2,'b')");
testSourceConfig(true, true);
testSourceConfig(false, false);
} finally {
TestBaseUtils.setEnv(env);
hiveShell.execute("drop database db1 cascade");
}
}
示例5
@Test
public void testWithEmptyGraphWithVertices() throws Exception {
DataSet<Vertex<LongValue, LongValue>> outDegreeWithoutZeroDegreeVertices = emptyGraphWithVertices
.run(new VertexOutDegree<LongValue, NullValue, NullValue>()
.setIncludeZeroDegreeVertices(false));
assertEquals(0, outDegreeWithoutZeroDegreeVertices.collect().size());
DataSet<Vertex<LongValue, LongValue>> outDegreeWithZeroDegreeVertices = emptyGraphWithVertices
.run(new VertexOutDegree<LongValue, NullValue, NullValue>()
.setIncludeZeroDegreeVertices(true));
String expectedResult =
"(0,0)\n" +
"(1,0)\n" +
"(2,0)";
TestBaseUtils.compareResultAsText(outDegreeWithZeroDegreeVertices.collect(), expectedResult);
}
示例6
@Test
public void testWithDirectedSimpleGraph() throws Exception {
DataSet<Vertex<IntValue, LongValue>> inDegree = directedSimpleGraph
.run(new VertexInDegree<IntValue, NullValue, NullValue>()
.setIncludeZeroDegreeVertices(true));
String expectedResult =
"(0,0)\n" +
"(1,3)\n" +
"(2,1)\n" +
"(3,2)\n" +
"(4,1)\n" +
"(5,0)";
TestBaseUtils.compareResultAsText(inDegree.collect(), expectedResult);
}
示例7
@Test
public void testWithSingletonEdgeGraph() throws Exception {
Graph<LongValue, Long, Double> result = new SingletonEdgeGraph(env, 1)
.generate()
.mapVertices(v -> v.getId().getValue(),
new TypeHint<Vertex<LongValue, Long>>(){}.getTypeInfo())
.mapEdges(e -> 1.0,
new TypeHint<Edge<LongValue, Double>>(){}.getTypeInfo())
.run(new CommunityDetection<>(10, 0.5));
String expectedResult =
"(0,0)\n" +
"(1,1)\n";
TestBaseUtils.compareResultAsText(result.getVertices().collect(), expectedResult);
}
示例8
@Test
public void testWithSimpleGraph() throws Exception {
Graph<IntValue, NullValue, NullValue> graph = undirectedSimpleGraph
.run(new MaximumDegree<>(3));
String expectedVerticesResult =
"(0,(null))\n" +
"(1,(null))\n" +
"(2,(null))\n" +
"(4,(null))\n" +
"(5,(null))";
TestBaseUtils.compareResultAsText(graph.getVertices().collect(), expectedVerticesResult);
String expectedEdgesResult =
"(0,1,(null))\n" +
"(0,2,(null))\n" +
"(1,0,(null))\n" +
"(1,2,(null))\n" +
"(2,0,(null))\n" +
"(2,1,(null))";
TestBaseUtils.compareResultAsText(graph.getEdges().collect(), expectedEdgesResult);
}
示例9
@Test
public void getTaskmanagers() throws Exception {
String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");
ObjectMapper mapper = new ObjectMapper();
JsonNode parsed = mapper.readTree(json);
ArrayNode taskManagers = (ArrayNode) parsed.get("taskmanagers");
assertNotNull(taskManagers);
assertEquals(NUM_TASK_MANAGERS, taskManagers.size());
JsonNode taskManager = taskManagers.get(0);
assertNotNull(taskManager);
assertEquals(NUM_SLOTS, taskManager.get("slotsNumber").asInt());
assertTrue(taskManager.get("freeSlots").asInt() <= NUM_SLOTS);
}
示例10
@Test
public void getNumberOfTaskManagers() {
try {
String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");
ObjectMapper mapper = new ObjectMapper();
JsonNode response = mapper.readTree(json);
ArrayNode taskManagers = (ArrayNode) response.get("taskmanagers");
assertNotNull(taskManagers);
assertEquals(NUM_TASK_MANAGERS, taskManagers.size());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
示例11
@Test
public void testWithEmptyGraphWithVertices() throws Exception {
DataSet<Vertex<LongValue, LongValue>> inDegreeWithoutZeroDegreeVertices = emptyGraphWithVertices
.run(new VertexInDegree<LongValue, NullValue, NullValue>()
.setIncludeZeroDegreeVertices(false));
assertEquals(0, inDegreeWithoutZeroDegreeVertices.collect().size());
DataSet<Vertex<LongValue, LongValue>> inDegreeWithZeroDegreeVertices = emptyGraphWithVertices
.run(new VertexInDegree<LongValue, NullValue, NullValue>()
.setIncludeZeroDegreeVertices(true));
String expectedResult =
"(0,0)\n" +
"(1,0)\n" +
"(2,0)";
TestBaseUtils.compareResultAsText(inDegreeWithZeroDegreeVertices.collect(), expectedResult);
}
示例12
@Test
public void testWithUndirectedSimpleGraph() throws Exception {
DataSet<Vertex<IntValue, LongValue>> inDegree = undirectedSimpleGraph
.run(new VertexInDegree<IntValue, NullValue, NullValue>()
.setIncludeZeroDegreeVertices(true));
String expectedResult =
"(0,2)\n" +
"(1,3)\n" +
"(2,3)\n" +
"(3,4)\n" +
"(4,1)\n" +
"(5,1)";
TestBaseUtils.compareResultAsText(inDegree.collect(), expectedResult);
}
示例13
@Test
public void testWithEmptyGraphWithVertices() throws Exception {
DataSet<Vertex<LongValue, Degrees>> degreesWithoutZeroDegreeVertices = emptyGraphWithVertices
.run(new VertexDegrees<LongValue, NullValue, NullValue>()
.setIncludeZeroDegreeVertices(false));
assertEquals(0, degreesWithoutZeroDegreeVertices.collect().size());
DataSet<Vertex<LongValue, Degrees>> degreesWithZeroDegreeVertices = emptyGraphWithVertices
.run(new VertexDegrees<LongValue, NullValue, NullValue>()
.setIncludeZeroDegreeVertices(true));
String expectedResult =
"(0,(0,0,0))\n" +
"(1,(0,0,0))\n" +
"(2,(0,0,0))";
TestBaseUtils.compareResultAsText(degreesWithZeroDegreeVertices.collect(), expectedResult);
}
示例14
@Test
public void testWithSimpleGraph() throws Exception {
String expectedResult =
"(0,1,((null),(3,0,3)))\n" +
"(0,2,((null),(3,2,1)))\n" +
"(2,1,((null),(3,0,3)))\n" +
"(2,3,((null),(4,2,2)))\n" +
"(3,1,((null),(3,0,3)))\n" +
"(3,4,((null),(1,0,1)))\n" +
"(5,3,((null),(4,2,2)))";
DataSet<Edge<IntValue, Tuple2<NullValue, Degrees>>> targetDegrees = directedSimpleGraph
.run(new EdgeTargetDegrees<>());
TestBaseUtils.compareResultAsText(targetDegrees.collect(), expectedResult);
}
示例15
@Test
public void testWithDirectedSimpleGraph() throws Exception {
DataSet<Vertex<IntValue, LongValue>> outDegree = directedSimpleGraph
.run(new VertexOutDegree<IntValue, NullValue, NullValue>()
.setIncludeZeroDegreeVertices(true));
String expectedResult =
"(0,2)\n" +
"(1,0)\n" +
"(2,2)\n" +
"(3,2)\n" +
"(4,0)\n" +
"(5,1)";
TestBaseUtils.compareResultAsText(outDegree.collect(), expectedResult);
}
示例16
@Test
public void testWithSimpleGraph() throws Exception {
String expectedResult =
"(0,1,((null),(2,2,0),(3,0,3)))\n" +
"(0,2,((null),(2,2,0),(3,2,1)))\n" +
"(2,1,((null),(3,2,1),(3,0,3)))\n" +
"(2,3,((null),(3,2,1),(4,2,2)))\n" +
"(3,1,((null),(4,2,2),(3,0,3)))\n" +
"(3,4,((null),(4,2,2),(1,0,1)))\n" +
"(5,3,((null),(1,1,0),(4,2,2)))";
DataSet<Edge<IntValue, Tuple3<NullValue, Degrees, Degrees>>> degreesPair = directedSimpleGraph
.run(new EdgeDegreesPair<>());
TestBaseUtils.compareResultAsText(degreesPair.collect(), expectedResult);
}
示例17
@Test
public void testWithSimpleGraph() throws Exception {
DataSet<Result<IntValue>> aa = undirectedSimpleGraph
.run(new AdamicAdar<>());
String expectedResult =
"(0,1," + ilog[2] + ")\n" +
"(0,2," + ilog[1] + ")\n" +
"(0,3," + (ilog[1] + ilog[2]) + ")\n" +
"(1,2," + (ilog[0] + ilog[3]) + ")\n" +
"(1,3," + ilog[2] + ")\n" +
"(1,4," + ilog[3] + ")\n" +
"(1,5," + ilog[3] + ")\n" +
"(2,3," + ilog[1] + ")\n" +
"(2,4," + ilog[3] + ")\n" +
"(2,5," + ilog[3] + ")\n" +
"(4,5," + ilog[3] + ")";
TestBaseUtils.compareResultAsText(aa.collect(), expectedResult);
}
示例18
@Test
public void testWithEmptyGraphWithVertices() throws Exception {
DataSet<Vertex<LongValue, LongValue>> degree;
degree = emptyGraphWithVertices
.run(new VertexDegree<LongValue, NullValue, NullValue>()
.setIncludeZeroDegreeVertices(false));
assertEquals(0, degree.collect().size());
degree = emptyGraphWithVertices
.run(new VertexDegree<LongValue, NullValue, NullValue>()
.setIncludeZeroDegreeVertices(true));
String expectedResult =
"(0,0)\n" +
"(1,0)\n" +
"(2,0)";
TestBaseUtils.compareResultAsText(degree.collect(), expectedResult);
}
示例19
@Test
public void testSimpleGraph() throws Exception {
DataSet<Result<IntValue>> ji = undirectedSimpleGraph
.run(new JaccardIndex<>());
String expectedResult =
"(0,1,1,4)\n" +
"(0,2,1,4)\n" +
"(0,3,2,4)\n" +
"(1,2,2,4)\n" +
"(1,3,1,6)\n" +
"(1,4,1,3)\n" +
"(1,5,1,3)\n" +
"(2,3,1,6)\n" +
"(2,4,1,3)\n" +
"(2,5,1,3)\n" +
"(4,5,1,1)\n";
TestBaseUtils.compareResultAsText(ji.collect(), expectedResult);
}
示例20
@Test
public void testWithSimpleGraphWithMaximumScore() throws Exception {
DataSet<Result<IntValue>> ji = undirectedSimpleGraph
.run(new JaccardIndex<IntValue, NullValue, NullValue>()
.setMaximumScore(1, 2));
String expectedResult =
"(0,1,1,4)\n" +
"(0,2,1,4)\n" +
"(0,3,2,4)\n" +
"(1,2,2,4)\n" +
"(1,3,1,6)\n" +
"(1,4,1,3)\n" +
"(1,5,1,3)\n" +
"(2,3,1,6)\n" +
"(2,4,1,3)\n" +
"(2,5,1,3)\n";
TestBaseUtils.compareResultAsText(ji.collect(), expectedResult);
}
示例21
@Test
public void testWithUndirectedSimpleGraph() throws Exception {
DataSet<Vertex<IntValue, LongValue>> outDegree = undirectedSimpleGraph
.run(new VertexOutDegree<IntValue, NullValue, NullValue>()
.setIncludeZeroDegreeVertices(true));
String expectedResult =
"(0,2)\n" +
"(1,3)\n" +
"(2,3)\n" +
"(3,4)\n" +
"(4,1)\n" +
"(5,1)";
TestBaseUtils.compareResultAsText(outDegree.collect(), expectedResult);
}
示例22
@Test
public void getTaskManagerLogAndStdoutFiles() {
try {
String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");
ObjectMapper mapper = new ObjectMapper();
JsonNode parsed = mapper.readTree(json);
ArrayNode taskManagers = (ArrayNode) parsed.get("taskmanagers");
JsonNode taskManager = taskManagers.get(0);
String id = taskManager.get("id").asText();
WebMonitorUtils.LogFileLocation logFiles = WebMonitorUtils.LogFileLocation.find(CLUSTER_CONFIGURATION);
//we check for job manager log files, since no separate taskmanager logs exist
FileUtils.writeStringToFile(logFiles.logFile, "job manager log");
String logs = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/" + id + "/log");
assertTrue(logs.contains("job manager log"));
FileUtils.writeStringToFile(logFiles.stdOutFile, "job manager out");
logs = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/" + id + "/stdout");
assertTrue(logs.contains("job manager out"));
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
示例23
@Test
public void testWithSimpleGraph() throws Exception {
Graph<IntValue, Long, Double> result = undirectedSimpleGraph
.mapVertices(v -> (long) v.getId().getValue(),
new TypeHint<Vertex<IntValue, Long>>(){}.getTypeInfo())
.mapEdges(e -> (double) e.getTarget().getValue() + e.getSource().getValue(),
new TypeHint<Edge<IntValue, Double>>(){}.getTypeInfo())
.run(new CommunityDetection<>(10, 0.5));
String expectedResult =
"(0,3)\n" +
"(1,5)\n" +
"(2,5)\n" +
"(3,3)\n" +
"(4,5)\n" +
"(5,5)\n";
TestBaseUtils.compareResultAsText(result.getVertices().collect(), expectedResult);
}
示例24
@Test
public void getNumberOfTaskManagers() {
try {
String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");
ObjectMapper mapper = new ObjectMapper();
JsonNode response = mapper.readTree(json);
ArrayNode taskManagers = (ArrayNode) response.get("taskmanagers");
assertNotNull(taskManagers);
assertEquals(NUM_TASK_MANAGERS, taskManagers.size());
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
示例25
@Test
public void getTaskmanagers() throws Exception {
String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");
ObjectMapper mapper = new ObjectMapper();
JsonNode parsed = mapper.readTree(json);
ArrayNode taskManagers = (ArrayNode) parsed.get("taskmanagers");
assertNotNull(taskManagers);
assertEquals(NUM_TASK_MANAGERS, taskManagers.size());
JsonNode taskManager = taskManagers.get(0);
assertNotNull(taskManager);
assertEquals(NUM_SLOTS, taskManager.get("slotsNumber").asInt());
assertTrue(taskManager.get("freeSlots").asInt() <= NUM_SLOTS);
}
示例26
@Test
public void getTaskManagerLogAndStdoutFiles() {
try {
String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");
ObjectMapper mapper = new ObjectMapper();
JsonNode parsed = mapper.readTree(json);
ArrayNode taskManagers = (ArrayNode) parsed.get("taskmanagers");
JsonNode taskManager = taskManagers.get(0);
String id = taskManager.get("id").asText();
WebMonitorUtils.LogFileLocation logFiles = WebMonitorUtils.LogFileLocation.find(CLUSTER_CONFIGURATION);
//we check for job manager log files, since no separate taskmanager logs exist
FileUtils.writeStringToFile(logFiles.logFile, "job manager log");
String logs = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/" + id + "/log");
assertTrue(logs.contains("job manager log"));
FileUtils.writeStringToFile(logFiles.stdOutFile, "job manager out");
logs = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/" + id + "/stdout");
assertTrue(logs.contains("job manager out"));
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
示例27
@Test
public void testWithEmptyGraphWithVertices() throws Exception {
DataSet<Vertex<LongValue, Degrees>> degreesWithoutZeroDegreeVertices = emptyGraphWithVertices
.run(new VertexDegrees<LongValue, NullValue, NullValue>()
.setIncludeZeroDegreeVertices(false));
assertEquals(0, degreesWithoutZeroDegreeVertices.collect().size());
DataSet<Vertex<LongValue, Degrees>> degreesWithZeroDegreeVertices = emptyGraphWithVertices
.run(new VertexDegrees<LongValue, NullValue, NullValue>()
.setIncludeZeroDegreeVertices(true));
String expectedResult =
"(0,(0,0,0))\n" +
"(1,(0,0,0))\n" +
"(2,(0,0,0))";
TestBaseUtils.compareResultAsText(degreesWithZeroDegreeVertices.collect(), expectedResult);
}
示例28
@Test
public void testWithSimpleGraphWithMinimumScore() throws Exception {
DataSet<Result<IntValue>> aa = undirectedSimpleGraph
.run(new AdamicAdar<IntValue, NullValue, NullValue>()
.setMinimumScore(0.75f));
String expectedResult =
"(0,1," + ilog[2] + ")\n" +
"(0,2," + ilog[1] + ")\n" +
"(0,3," + (ilog[1] + ilog[2]) + ")\n" +
"(1,2," + (ilog[0] + ilog[3]) + ")\n" +
"(1,3," + ilog[2] + ")\n" +
"(2,3," + ilog[1] + ")";
TestBaseUtils.compareResultAsText(aa.collect(), expectedResult);
}
示例29
@Test
public void testWithSimpleGraph() throws Exception {
Graph<IntValue, Long, Double> result = undirectedSimpleGraph
.mapVertices(v -> (long) v.getId().getValue(),
new TypeHint<Vertex<IntValue, Long>>(){}.getTypeInfo())
.mapEdges(e -> (double) e.getTarget().getValue() + e.getSource().getValue(),
new TypeHint<Edge<IntValue, Double>>(){}.getTypeInfo())
.run(new CommunityDetection<>(10, 0.5));
String expectedResult =
"(0,3)\n" +
"(1,5)\n" +
"(2,5)\n" +
"(3,3)\n" +
"(4,5)\n" +
"(5,5)\n";
TestBaseUtils.compareResultAsText(result.getVertices().collect(), expectedResult);
}
示例30
@Test
public void testTableSourceFullScan() throws Exception {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(4);
BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, new TableConfig());
HBaseTableSource hbaseTable = new HBaseTableSource(getConf(), TEST_TABLE);
hbaseTable.addColumn(FAMILY1, F1COL1, Integer.class);
hbaseTable.addColumn(FAMILY2, F2COL1, String.class);
hbaseTable.addColumn(FAMILY2, F2COL2, Long.class);
hbaseTable.addColumn(FAMILY3, F3COL1, Double.class);
hbaseTable.addColumn(FAMILY3, F3COL2, Boolean.class);
hbaseTable.addColumn(FAMILY3, F3COL3, String.class);
tableEnv.registerTableSource("hTable", hbaseTable);
Table result = tableEnv.sqlQuery(
"SELECT " +
" h.family1.col1, " +
" h.family2.col1, " +
" h.family2.col2, " +
" h.family3.col1, " +
" h.family3.col2, " +
" h.family3.col3 " +
"FROM hTable AS h"
);
DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
List<Row> results = resultSet.collect();
String expected =
"10,Hello-1,100,1.01,false,Welt-1\n" +
"20,Hello-2,200,2.02,true,Welt-2\n" +
"30,Hello-3,300,3.03,false,Welt-3\n" +
"40,null,400,4.04,true,Welt-4\n" +
"50,Hello-5,500,5.05,false,Welt-5\n" +
"60,Hello-6,600,6.06,true,Welt-6\n" +
"70,Hello-7,700,7.07,false,Welt-7\n" +
"80,null,800,8.08,true,Welt-8\n";
TestBaseUtils.compareResultAsText(results, expected);
}