@After
public void after() throws Exception {
	TestBaseUtils.compareResultsByLinesInMemory(expectedTopSongs, topSongsResultPath);

	ArrayList<String> list = new ArrayList<>();
	TestBaseUtils.readAllResultLines(list, communitiesResultPath, new String[]{}, false);

	String[] result = list.toArray(new String[list.size()]);
	Arrays.sort(result);

	// check that user_1 and user_2 are in the same community
	Assert.assertEquals("users 1 and 2 are not in the same community",
			result[0].substring(7), result[1].substring(7));

	// check that user_3, user_4 and user_5 are in the same community
	Assert.assertEquals("users 3 and 4 are not in the same community",
			result[2].substring(7), result[3].substring(7));
	Assert.assertEquals("users 4 and 5 are not in the same community",
			result[3].substring(7), result[4].substring(7));
}
 

@Test
public void testSimpleGraph() throws Exception {
	DataSet<Result<IntValue>> ji = undirectedSimpleGraph
		.run(new JaccardIndex<>());

	String expectedResult =
		"(0,1,1,4)\n" +
		"(0,2,1,4)\n" +
		"(0,3,2,4)\n" +
		"(1,2,2,4)\n" +
		"(1,3,1,6)\n" +
		"(1,4,1,3)\n" +
		"(1,5,1,3)\n" +
		"(2,3,1,6)\n" +
		"(2,4,1,3)\n" +
		"(2,5,1,3)\n" +
		"(4,5,1,1)\n";

	TestBaseUtils.compareResultAsText(ji.collect(), expectedResult);
}
 

@Test
public void testWithSimpleGraph() throws Exception {
	String expectedResult =
		"(0,1,((null),(2,2,0)))\n" +
		"(0,2,((null),(2,2,0)))\n" +
		"(2,1,((null),(3,2,1)))\n" +
		"(2,3,((null),(3,2,1)))\n" +
		"(3,1,((null),(4,2,2)))\n" +
		"(3,4,((null),(4,2,2)))\n" +
		"(5,3,((null),(1,1,0)))";

	DataSet<Edge<IntValue, Tuple2<NullValue, Degrees>>> sourceDegrees = directedSimpleGraph
			.run(new EdgeSourceDegrees<>());

	TestBaseUtils.compareResultAsText(sourceDegrees.collect(), expectedResult);
}
 

@Test
public void testSourceConfig() throws Exception {
	// vector reader not available for 1.x and we're not testing orc for 2.0.x
	Assume.assumeTrue(HiveVersionTestUtil.HIVE_210_OR_LATER);
	Map<String, String> env = System.getenv();
	hiveShell.execute("create database db1");
	try {
		hiveShell.execute("create table db1.src (x int,y string) stored as orc");
		hiveShell.execute("insert into db1.src values (1,'a'),(2,'b')");
		testSourceConfig(true, true);
		testSourceConfig(false, false);
	} finally {
		TestBaseUtils.setEnv(env);
		hiveShell.execute("drop database db1 cascade");
	}
}
 

@Test
public void testWithEmptyGraphWithVertices() throws Exception {
	DataSet<Vertex<LongValue, LongValue>> outDegreeWithoutZeroDegreeVertices = emptyGraphWithVertices
		.run(new VertexOutDegree<LongValue, NullValue, NullValue>()
			.setIncludeZeroDegreeVertices(false));

	assertEquals(0, outDegreeWithoutZeroDegreeVertices.collect().size());

	DataSet<Vertex<LongValue, LongValue>> outDegreeWithZeroDegreeVertices = emptyGraphWithVertices
		.run(new VertexOutDegree<LongValue, NullValue, NullValue>()
			.setIncludeZeroDegreeVertices(true));

	String expectedResult =
		"(0,0)\n" +
		"(1,0)\n" +
		"(2,0)";

	TestBaseUtils.compareResultAsText(outDegreeWithZeroDegreeVertices.collect(), expectedResult);
}
 

@Test
public void testWithDirectedSimpleGraph() throws Exception {
	DataSet<Vertex<IntValue, LongValue>> inDegree = directedSimpleGraph
		.run(new VertexInDegree<IntValue, NullValue, NullValue>()
			.setIncludeZeroDegreeVertices(true));

	String expectedResult =
		"(0,0)\n" +
		"(1,3)\n" +
		"(2,1)\n" +
		"(3,2)\n" +
		"(4,1)\n" +
		"(5,0)";

	TestBaseUtils.compareResultAsText(inDegree.collect(), expectedResult);
}
 

@Test
public void testWithSingletonEdgeGraph() throws Exception {
	Graph<LongValue, Long, Double> result = new SingletonEdgeGraph(env, 1)
		.generate()
		.mapVertices(v -> v.getId().getValue(),
			new TypeHint<Vertex<LongValue, Long>>(){}.getTypeInfo())
		.mapEdges(e -> 1.0,
			new TypeHint<Edge<LongValue, Double>>(){}.getTypeInfo())
		.run(new CommunityDetection<>(10, 0.5));

	String expectedResult =
		"(0,0)\n" +
		"(1,1)\n";

	TestBaseUtils.compareResultAsText(result.getVertices().collect(), expectedResult);
}
 

@Test
public void testWithSimpleGraph() throws Exception {
	Graph<IntValue, NullValue, NullValue> graph = undirectedSimpleGraph
		.run(new MaximumDegree<>(3));

	String expectedVerticesResult =
		"(0,(null))\n" +
		"(1,(null))\n" +
		"(2,(null))\n" +
		"(4,(null))\n" +
		"(5,(null))";

	TestBaseUtils.compareResultAsText(graph.getVertices().collect(), expectedVerticesResult);

	String expectedEdgesResult =
		"(0,1,(null))\n" +
		"(0,2,(null))\n" +
		"(1,0,(null))\n" +
		"(1,2,(null))\n" +
		"(2,0,(null))\n" +
		"(2,1,(null))";

	TestBaseUtils.compareResultAsText(graph.getEdges().collect(), expectedEdgesResult);
}
 

@Test
public void getTaskmanagers() throws Exception {
	String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");

	ObjectMapper mapper = new ObjectMapper();
	JsonNode parsed = mapper.readTree(json);
	ArrayNode taskManagers = (ArrayNode) parsed.get("taskmanagers");

	assertNotNull(taskManagers);
	assertEquals(NUM_TASK_MANAGERS, taskManagers.size());

	JsonNode taskManager = taskManagers.get(0);
	assertNotNull(taskManager);
	assertEquals(NUM_SLOTS, taskManager.get("slotsNumber").asInt());
	assertTrue(taskManager.get("freeSlots").asInt() <= NUM_SLOTS);
}
 

@Test
public void getNumberOfTaskManagers() {
	try {
		String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");

		ObjectMapper mapper = new ObjectMapper();
		JsonNode response = mapper.readTree(json);
		ArrayNode taskManagers = (ArrayNode) response.get("taskmanagers");

		assertNotNull(taskManagers);
		assertEquals(NUM_TASK_MANAGERS, taskManagers.size());
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testWithEmptyGraphWithVertices() throws Exception {
	DataSet<Vertex<LongValue, LongValue>> inDegreeWithoutZeroDegreeVertices = emptyGraphWithVertices
		.run(new VertexInDegree<LongValue, NullValue, NullValue>()
			.setIncludeZeroDegreeVertices(false));

	assertEquals(0, inDegreeWithoutZeroDegreeVertices.collect().size());

	DataSet<Vertex<LongValue, LongValue>> inDegreeWithZeroDegreeVertices = emptyGraphWithVertices
		.run(new VertexInDegree<LongValue, NullValue, NullValue>()
			.setIncludeZeroDegreeVertices(true));

	String expectedResult =
		"(0,0)\n" +
		"(1,0)\n" +
		"(2,0)";

	TestBaseUtils.compareResultAsText(inDegreeWithZeroDegreeVertices.collect(), expectedResult);
}
 

@Test
public void testWithUndirectedSimpleGraph() throws Exception {
	DataSet<Vertex<IntValue, LongValue>> inDegree = undirectedSimpleGraph
		.run(new VertexInDegree<IntValue, NullValue, NullValue>()
			.setIncludeZeroDegreeVertices(true));

	String expectedResult =
		"(0,2)\n" +
		"(1,3)\n" +
		"(2,3)\n" +
		"(3,4)\n" +
		"(4,1)\n" +
		"(5,1)";

	TestBaseUtils.compareResultAsText(inDegree.collect(), expectedResult);
}
 

@Test
public void testWithEmptyGraphWithVertices() throws Exception {
	DataSet<Vertex<LongValue, Degrees>> degreesWithoutZeroDegreeVertices = emptyGraphWithVertices
		.run(new VertexDegrees<LongValue, NullValue, NullValue>()
			.setIncludeZeroDegreeVertices(false));

	assertEquals(0, degreesWithoutZeroDegreeVertices.collect().size());

	DataSet<Vertex<LongValue, Degrees>> degreesWithZeroDegreeVertices = emptyGraphWithVertices
		.run(new VertexDegrees<LongValue, NullValue, NullValue>()
			.setIncludeZeroDegreeVertices(true));

	String expectedResult =
		"(0,(0,0,0))\n" +
		"(1,(0,0,0))\n" +
		"(2,(0,0,0))";

	TestBaseUtils.compareResultAsText(degreesWithZeroDegreeVertices.collect(), expectedResult);
}
 

@Test
public void testWithSimpleGraph() throws Exception {
	String expectedResult =
		"(0,1,((null),(3,0,3)))\n" +
		"(0,2,((null),(3,2,1)))\n" +
		"(2,1,((null),(3,0,3)))\n" +
		"(2,3,((null),(4,2,2)))\n" +
		"(3,1,((null),(3,0,3)))\n" +
		"(3,4,((null),(1,0,1)))\n" +
		"(5,3,((null),(4,2,2)))";

	DataSet<Edge<IntValue, Tuple2<NullValue, Degrees>>> targetDegrees = directedSimpleGraph
			.run(new EdgeTargetDegrees<>());

	TestBaseUtils.compareResultAsText(targetDegrees.collect(), expectedResult);
}
 

@Test
public void testWithDirectedSimpleGraph() throws Exception {
	DataSet<Vertex<IntValue, LongValue>> outDegree = directedSimpleGraph
		.run(new VertexOutDegree<IntValue, NullValue, NullValue>()
			.setIncludeZeroDegreeVertices(true));

	String expectedResult =
		"(0,2)\n" +
		"(1,0)\n" +
		"(2,2)\n" +
		"(3,2)\n" +
		"(4,0)\n" +
		"(5,1)";

	TestBaseUtils.compareResultAsText(outDegree.collect(), expectedResult);
}
 

@Test
public void testWithSimpleGraph() throws Exception {
	String expectedResult =
		"(0,1,((null),(2,2,0),(3,0,3)))\n" +
		"(0,2,((null),(2,2,0),(3,2,1)))\n" +
		"(2,1,((null),(3,2,1),(3,0,3)))\n" +
		"(2,3,((null),(3,2,1),(4,2,2)))\n" +
		"(3,1,((null),(4,2,2),(3,0,3)))\n" +
		"(3,4,((null),(4,2,2),(1,0,1)))\n" +
		"(5,3,((null),(1,1,0),(4,2,2)))";

	DataSet<Edge<IntValue, Tuple3<NullValue, Degrees, Degrees>>> degreesPair = directedSimpleGraph
		.run(new EdgeDegreesPair<>());

	TestBaseUtils.compareResultAsText(degreesPair.collect(), expectedResult);
}
 

@Test
public void testWithSimpleGraph() throws Exception {
	DataSet<Result<IntValue>> aa = undirectedSimpleGraph
		.run(new AdamicAdar<>());

	String expectedResult =
		"(0,1," + ilog[2] + ")\n" +
		"(0,2," + ilog[1] + ")\n" +
		"(0,3," + (ilog[1] + ilog[2]) + ")\n" +
		"(1,2," + (ilog[0] + ilog[3]) + ")\n" +
		"(1,3," + ilog[2] + ")\n" +
		"(1,4," + ilog[3] + ")\n" +
		"(1,5," + ilog[3] + ")\n" +
		"(2,3," + ilog[1] + ")\n" +
		"(2,4," + ilog[3] + ")\n" +
		"(2,5," + ilog[3] + ")\n" +
		"(4,5," + ilog[3] + ")";

	TestBaseUtils.compareResultAsText(aa.collect(), expectedResult);
}
 

@Test
public void testWithEmptyGraphWithVertices() throws Exception {
	DataSet<Vertex<LongValue, LongValue>> degree;

	degree = emptyGraphWithVertices
		.run(new VertexDegree<LongValue, NullValue, NullValue>()
			.setIncludeZeroDegreeVertices(false));

	assertEquals(0, degree.collect().size());

	degree = emptyGraphWithVertices
		.run(new VertexDegree<LongValue, NullValue, NullValue>()
			.setIncludeZeroDegreeVertices(true));

	String expectedResult =
		"(0,0)\n" +
		"(1,0)\n" +
		"(2,0)";

	TestBaseUtils.compareResultAsText(degree.collect(), expectedResult);
}
 

@Test
public void testSimpleGraph() throws Exception {
	DataSet<Result<IntValue>> ji = undirectedSimpleGraph
		.run(new JaccardIndex<>());

	String expectedResult =
		"(0,1,1,4)\n" +
		"(0,2,1,4)\n" +
		"(0,3,2,4)\n" +
		"(1,2,2,4)\n" +
		"(1,3,1,6)\n" +
		"(1,4,1,3)\n" +
		"(1,5,1,3)\n" +
		"(2,3,1,6)\n" +
		"(2,4,1,3)\n" +
		"(2,5,1,3)\n" +
		"(4,5,1,1)\n";

	TestBaseUtils.compareResultAsText(ji.collect(), expectedResult);
}
 

@Test
public void testWithSimpleGraphWithMaximumScore() throws Exception {
	DataSet<Result<IntValue>> ji = undirectedSimpleGraph
		.run(new JaccardIndex<IntValue, NullValue, NullValue>()
			.setMaximumScore(1, 2));

	String expectedResult =
		"(0,1,1,4)\n" +
		"(0,2,1,4)\n" +
		"(0,3,2,4)\n" +
		"(1,2,2,4)\n" +
		"(1,3,1,6)\n" +
		"(1,4,1,3)\n" +
		"(1,5,1,3)\n" +
		"(2,3,1,6)\n" +
		"(2,4,1,3)\n" +
		"(2,5,1,3)\n";

	TestBaseUtils.compareResultAsText(ji.collect(), expectedResult);
}
 

@Test
public void testWithUndirectedSimpleGraph() throws Exception {
	DataSet<Vertex<IntValue, LongValue>> outDegree = undirectedSimpleGraph
		.run(new VertexOutDegree<IntValue, NullValue, NullValue>()
			.setIncludeZeroDegreeVertices(true));

	String expectedResult =
		"(0,2)\n" +
		"(1,3)\n" +
		"(2,3)\n" +
		"(3,4)\n" +
		"(4,1)\n" +
		"(5,1)";

	TestBaseUtils.compareResultAsText(outDegree.collect(), expectedResult);
}
 

@Test
public void getTaskManagerLogAndStdoutFiles() {
	try {
		String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");

		ObjectMapper mapper = new ObjectMapper();
		JsonNode parsed = mapper.readTree(json);
		ArrayNode taskManagers = (ArrayNode) parsed.get("taskmanagers");
		JsonNode taskManager = taskManagers.get(0);
		String id = taskManager.get("id").asText();

		WebMonitorUtils.LogFileLocation logFiles = WebMonitorUtils.LogFileLocation.find(CLUSTER_CONFIGURATION);

		//we check for job manager log files, since no separate taskmanager logs exist
		FileUtils.writeStringToFile(logFiles.logFile, "job manager log");
		String logs = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/" + id + "/log");
		assertTrue(logs.contains("job manager log"));

		FileUtils.writeStringToFile(logFiles.stdOutFile, "job manager out");
		logs = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/" + id + "/stdout");
		assertTrue(logs.contains("job manager out"));
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testWithSimpleGraph() throws Exception {
	Graph<IntValue, Long, Double> result = undirectedSimpleGraph
		.mapVertices(v -> (long) v.getId().getValue(),
			new TypeHint<Vertex<IntValue, Long>>(){}.getTypeInfo())
		.mapEdges(e -> (double) e.getTarget().getValue() + e.getSource().getValue(),
			new TypeHint<Edge<IntValue, Double>>(){}.getTypeInfo())
		.run(new CommunityDetection<>(10, 0.5));

	String expectedResult =
		"(0,3)\n" +
		"(1,5)\n" +
		"(2,5)\n" +
		"(3,3)\n" +
		"(4,5)\n" +
		"(5,5)\n";

	TestBaseUtils.compareResultAsText(result.getVertices().collect(), expectedResult);
}
 

@Test
public void getNumberOfTaskManagers() {
	try {
		String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");

		ObjectMapper mapper = new ObjectMapper();
		JsonNode response = mapper.readTree(json);
		ArrayNode taskManagers = (ArrayNode) response.get("taskmanagers");

		assertNotNull(taskManagers);
		assertEquals(NUM_TASK_MANAGERS, taskManagers.size());
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void getTaskmanagers() throws Exception {
	String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");

	ObjectMapper mapper = new ObjectMapper();
	JsonNode parsed = mapper.readTree(json);
	ArrayNode taskManagers = (ArrayNode) parsed.get("taskmanagers");

	assertNotNull(taskManagers);
	assertEquals(NUM_TASK_MANAGERS, taskManagers.size());

	JsonNode taskManager = taskManagers.get(0);
	assertNotNull(taskManager);
	assertEquals(NUM_SLOTS, taskManager.get("slotsNumber").asInt());
	assertTrue(taskManager.get("freeSlots").asInt() <= NUM_SLOTS);
}
 

@Test
public void getTaskManagerLogAndStdoutFiles() {
	try {
		String json = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/");

		ObjectMapper mapper = new ObjectMapper();
		JsonNode parsed = mapper.readTree(json);
		ArrayNode taskManagers = (ArrayNode) parsed.get("taskmanagers");
		JsonNode taskManager = taskManagers.get(0);
		String id = taskManager.get("id").asText();

		WebMonitorUtils.LogFileLocation logFiles = WebMonitorUtils.LogFileLocation.find(CLUSTER_CONFIGURATION);

		//we check for job manager log files, since no separate taskmanager logs exist
		FileUtils.writeStringToFile(logFiles.logFile, "job manager log");
		String logs = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/" + id + "/log");
		assertTrue(logs.contains("job manager log"));

		FileUtils.writeStringToFile(logFiles.stdOutFile, "job manager out");
		logs = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/taskmanagers/" + id + "/stdout");
		assertTrue(logs.contains("job manager out"));
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testWithEmptyGraphWithVertices() throws Exception {
	DataSet<Vertex<LongValue, Degrees>> degreesWithoutZeroDegreeVertices = emptyGraphWithVertices
		.run(new VertexDegrees<LongValue, NullValue, NullValue>()
			.setIncludeZeroDegreeVertices(false));

	assertEquals(0, degreesWithoutZeroDegreeVertices.collect().size());

	DataSet<Vertex<LongValue, Degrees>> degreesWithZeroDegreeVertices = emptyGraphWithVertices
		.run(new VertexDegrees<LongValue, NullValue, NullValue>()
			.setIncludeZeroDegreeVertices(true));

	String expectedResult =
		"(0,(0,0,0))\n" +
		"(1,(0,0,0))\n" +
		"(2,(0,0,0))";

	TestBaseUtils.compareResultAsText(degreesWithZeroDegreeVertices.collect(), expectedResult);
}
 

@Test
public void testWithSimpleGraphWithMinimumScore() throws Exception {
	DataSet<Result<IntValue>> aa = undirectedSimpleGraph
		.run(new AdamicAdar<IntValue, NullValue, NullValue>()
			.setMinimumScore(0.75f));

	String expectedResult =
		"(0,1," + ilog[2] + ")\n" +
		"(0,2," + ilog[1] + ")\n" +
		"(0,3," + (ilog[1] + ilog[2]) + ")\n" +
		"(1,2," + (ilog[0] + ilog[3]) + ")\n" +
		"(1,3," + ilog[2] + ")\n" +
		"(2,3," + ilog[1] + ")";

	TestBaseUtils.compareResultAsText(aa.collect(), expectedResult);
}
 

@Test
public void testWithSimpleGraph() throws Exception {
	Graph<IntValue, Long, Double> result = undirectedSimpleGraph
		.mapVertices(v -> (long) v.getId().getValue(),
			new TypeHint<Vertex<IntValue, Long>>(){}.getTypeInfo())
		.mapEdges(e -> (double) e.getTarget().getValue() + e.getSource().getValue(),
			new TypeHint<Edge<IntValue, Double>>(){}.getTypeInfo())
		.run(new CommunityDetection<>(10, 0.5));

	String expectedResult =
		"(0,3)\n" +
		"(1,5)\n" +
		"(2,5)\n" +
		"(3,3)\n" +
		"(4,5)\n" +
		"(5,5)\n";

	TestBaseUtils.compareResultAsText(result.getVertices().collect(), expectedResult);
}
 

@Test
public void testTableSourceFullScan() throws Exception {

	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(4);
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, new TableConfig());
	HBaseTableSource hbaseTable = new HBaseTableSource(getConf(), TEST_TABLE);
	hbaseTable.addColumn(FAMILY1, F1COL1, Integer.class);
	hbaseTable.addColumn(FAMILY2, F2COL1, String.class);
	hbaseTable.addColumn(FAMILY2, F2COL2, Long.class);
	hbaseTable.addColumn(FAMILY3, F3COL1, Double.class);
	hbaseTable.addColumn(FAMILY3, F3COL2, Boolean.class);
	hbaseTable.addColumn(FAMILY3, F3COL3, String.class);
	tableEnv.registerTableSource("hTable", hbaseTable);

	Table result = tableEnv.sqlQuery(
		"SELECT " +
			"  h.family1.col1, " +
			"  h.family2.col1, " +
			"  h.family2.col2, " +
			"  h.family3.col1, " +
			"  h.family3.col2, " +
			"  h.family3.col3 " +
			"FROM hTable AS h"
	);
	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();

	String expected =
		"10,Hello-1,100,1.01,false,Welt-1\n" +
		"20,Hello-2,200,2.02,true,Welt-2\n" +
		"30,Hello-3,300,3.03,false,Welt-3\n" +
		"40,null,400,4.04,true,Welt-4\n" +
		"50,Hello-5,500,5.05,false,Welt-5\n" +
		"60,Hello-6,600,6.06,true,Welt-6\n" +
		"70,Hello-7,700,7.07,false,Welt-7\n" +
		"80,null,800,8.08,true,Welt-8\n";

	TestBaseUtils.compareResultAsText(results, expected);
}