linux build now working; TODO: replace harcoded values in unsuck_platform_specific.cpp

.gitignore

@@ -3,4 +3,5 @@ Converter/bin
 Converter/.vs
 Converter/obj
 Converter.vcxproj.user
-nocommit
+nocommit
+build

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.16)
 set(CMAKE_SUPPRESS_REGENERATION true)
 
-project(PotreeConverter)
+project(PotreeConverter LANGUAGES CXX)
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
@@ -46,6 +46,19 @@ target_include_directories(PotreeConverter PRIVATE "./Converter/include")
 target_include_directories(PotreeConverter PRIVATE "./Converter/modules")
 target_include_directories(PotreeConverter PRIVATE "./Converter/libs")
 
+
+if (UNIX)
+	find_package(Threads REQUIRED)
+	find_package(TBB REQUIRED)
+	
+	target_link_libraries(${PROJECT_NAME} Threads::Threads)
+	target_link_libraries(${PROJECT_NAME} tbb)
+
+
+	#SET(CMAKE_CXX_FLAGS "-pthread -ltbb")
+endif (UNIX)
+
+
 ###############################################
 # COPY LICENSE FILES TO BINARY DIRECTORY
 ###############################################

Converter/include/Attributes.h

@@ -172,6 +172,8 @@ struct Attributes {
 				return &attribute;
 			}
 		}
+		
+		return nullptr;
 	}
 
 };

Converter/include/indexer.h

@@ -14,6 +14,7 @@
 #include <thread>
 #include <chrono>
 #include <mutex>
+#include <condition_variable>
 
 #include "json/json.hpp"
 

Converter/include/indexer_poissondisk.h

@@ -14,6 +14,7 @@
 #include <thread>
 #include <chrono>
 #include <mutex>
+#include <condition_variable>
 
 #include "json/json.hpp"
 

Converter/include/indexer_random.h

@@ -14,6 +14,7 @@
 #include <thread>
 #include <chrono>
 #include <mutex>
+#include <condition_variable>
 
 #include "json/json.hpp"
 

Converter/include/sampler_random.h

@@ -73,9 +73,9 @@ struct SamplerRandom : public Sampler {
 				int64_t y = double(gridSize) * ny;
 				int64_t z = double(gridSize) * nz;
 
-				x = std::max(0ll, std::min(x, gridSize - 1));
-				y = std::max(0ll, std::min(y, gridSize - 1));
-				z = std::max(0ll, std::min(z, gridSize - 1));
+				x = std::max(int64_t(0), std::min(x, gridSize - 1));
+				y = std::max(int64_t(0), std::min(y, gridSize - 1));
+				z = std::max(int64_t(0), std::min(z, gridSize - 1));
 
 				int64_t index = x + y * gridSize + z * gridSize * gridSize;
 

Converter/libs/laszip/CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.16)
 set(CMAKE_SUPPRESS_REGENERATION true)
 
-project(LASzip)
+project(LASzip LANGUAGES CXX)
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED True)

Converter/modules/unsuck/unsuck.hpp

@@ -14,6 +14,7 @@
 #include <algorithm>
 #include <thread>
 #include <cstdint>
+#include <cstring>
 
 using std::cout;
 using std::endl;
@@ -37,6 +38,10 @@ static long long unsuck_start_time = high_resolution_clock::now().time_since_epo
 
 static double Infinity = std::numeric_limits<double>::infinity();
 
+
+#define _fseeki64 fseeko64
+
+
 struct MemoryData {
 	size_t virtual_total = 0;
 	size_t virtual_used = 0;
@@ -541,7 +546,8 @@ T read(vector<uint8_t>& buffer, int offset) {
 
 inline string leftPad(string in, int length, const char character = ' ') {
 
-	auto reps = std::max(length - in.size(), 0ull);
+	int tmp = length - in.size();
+	auto reps = std::max(tmp, 0);
 	string result = string(reps, character) + in;
 
 	return result;

Converter/modules/unsuck/unsuck_platform_specific.cpp

@@ -164,4 +164,93 @@ CpuData getCpuData() {
 	return data;
 }
 
+#elif defined(__linux__)
+
+// see https://stackoverflow.com/questions/63166/how-to-determine-cpu-and-memory-consumption-from-inside-a-process
+MemoryData getMemoryData() {
+
+	MemoryData data;
+
+	{
+		data.virtual_total = 10'000'000'000;
+		data.virtual_used = 2'000'000'000;
+		data.physical_total = 10'000'000'000;
+		data.physical_used = 2'000'000'000;
+
+	}
+
+	{
+		data.virtual_usedByProcess = 10'000'000'000;
+		data.virtual_usedByProcess_max = 2'000'000'000;
+		data.physical_usedByProcess = 10'000'000'000;
+		data.physical_usedByProcess_max = 2'000'000'000;
+	}
+
+
+	return data;
+}
+
+
+void printMemoryReport() {
+
+	auto memoryData = getMemoryData();
+	double vm = double(memoryData.virtual_usedByProcess) / (1024.0 * 1024.0 * 1024.0);
+	double pm = double(memoryData.physical_usedByProcess) / (1024.0 * 1024.0 * 1024.0);
+
+	stringstream ss;
+	ss << "memory usage: "
+		<< "virtual: " << formatNumber(vm, 1) << " GB, "
+		<< "physical: " << formatNumber(pm, 1) << " GB"
+		<< endl;
+
+	cout << ss.str();
+
+}
+
+void launchMemoryChecker(int64_t maxMB, double checkInterval) {
+
+	auto interval = std::chrono::milliseconds(int64_t(checkInterval * 1000));
+
+	thread t([maxMB, interval]() {
+
+		static double lastReport = 0.0;
+		static double reportInterval = 1.0;
+		static double lastUsage = 0.0;
+		static double largestUsage = 0.0;
+
+		while (true) {
+			auto memdata = getMemoryData();
+
+			using namespace std::chrono_literals;
+			std::this_thread::sleep_for(interval);
+		}
+
+	});
+	t.detach();
+
+}
+
+static int numProcessors;
+static bool initialized = false;
+
+void init() {
+	numProcessors = 32;
+
+	initialized = true;
+}
+
+CpuData getCpuData() {
+	
+	if (!initialized) {
+		init();
+	}
+
+	CpuData data;
+	data.numProcessors = numProcessors;
+	data.usage = 50;
+
+	return data;
+}
+
+
 #endif

Converter/src/chunker_countsort_laszip.cpp

@@ -159,11 +159,6 @@ namespace chunker_countsort_laszip {
 			Vector3 min = task->min;
 			Vector3 max = task->max;
 
-			//static int dbg = 0;
-			//int dbgCurr = dbg;
-			//dbg++;
-			//cout << ("start: " + formatNumber(dbgCurr)) << endl;
-
 			thread_local unique_ptr<void, void(*)(void*)> buffer(nullptr, free);
 			thread_local int64_t bufferSize = -1;
 
@@ -662,7 +657,6 @@ namespace chunker_countsort_laszip {
 			thread_local int64_t bufferSize = -1;
 
 			if (bufferSize < numBytes) {
-
 				buffer.reset(malloc(numBytes));
 				bufferSize = numBytes;
 			}
@@ -825,7 +819,7 @@ namespace chunker_countsort_laszip {
 
 		};
 
-		TaskPool<Task> pool(12, processor);
+		TaskPool<Task> pool(numChunkerThreads, processor);
 
 		for (auto source: sources) {
 
@@ -1077,7 +1071,8 @@ namespace chunker_countsort_laszip {
 
 		auto tStart = now();
 
-		maxPointsPerChunk = std::min(state.pointsTotal / 20, 10'000'000ll);
+		int64_t tmp = state.pointsTotal / 20;
+		maxPointsPerChunk = std::min(tmp, int64_t(10'000'000));
 		cout << "maxPointsPerChunk: " << maxPointsPerChunk << endl;
 
 		if (state.pointsTotal < 100'000'000) {

Converter/src/indexer.cpp

@@ -761,9 +761,9 @@ void buildHierarchy(Indexer* indexer, Node* node, shared_ptr<Buffer> points, int
 		int64_t iy = double(counterGridSize) * (y - min.y) / size.y;
 		int64_t iz = double(counterGridSize) * (z - min.z) / size.z;
 
-		ix = std::max(0ll, std::min(ix, counterGridSize - 1));
-		iy = std::max(0ll, std::min(iy, counterGridSize - 1));
-		iz = std::max(0ll, std::min(iz, counterGridSize - 1));
+		ix = std::max(int64_t(0), std::min(ix, counterGridSize - 1));
+		iy = std::max(int64_t(0), std::min(iy, counterGridSize - 1));
+		iz = std::max(int64_t(0), std::min(iz, counterGridSize - 1));
 
 		int64_t index = mortonEncode_magicbits(iz, iy, ix);
 

Converter/src/indexer_poissondisk.cpp

@@ -783,9 +783,9 @@ void buildHierarchy(Indexer* indexer, Node* node, shared_ptr<Buffer> points, int
 		int64_t iy = double(counterGridSize) * (y - min.y) / size.y;
 		int64_t iz = double(counterGridSize) * (z - min.z) / size.z;
 
-		ix = std::max(0ll, std::min(ix, counterGridSize - 1));
-		iy = std::max(0ll, std::min(iy, counterGridSize - 1));
-		iz = std::max(0ll, std::min(iz, counterGridSize - 1));
+		ix = std::max(int64_t(0), std::min(ix, counterGridSize - 1));
+		iy = std::max(int64_t(0), std::min(iy, counterGridSize - 1));
+		iz = std::max(int64_t(0), std::min(iz, counterGridSize - 1));
 
 		int64_t index = mortonEncode_magicbits(iz, iy, ix);
 

Converter/src/indexer_random.cpp

@@ -259,9 +259,9 @@ void Indexer::doSampling(shared_ptr<Node> node, int64_t dbgShouldWriteBytes) {
 			int64_t y = double(gridSize) * ny;
 			int64_t z = double(gridSize) * nz;
 
-			x = std::max(0ll, std::min(x, gridSize - 1));
-			y = std::max(0ll, std::min(y, gridSize - 1));
-			z = std::max(0ll, std::min(z, gridSize - 1));
+			x = std::max(int64_t(0), std::min(x, gridSize - 1));
+			y = std::max(int64_t(0), std::min(y, gridSize - 1));
+			z = std::max(int64_t(0), std::min(z, gridSize - 1));
 
 			int64_t index = x + y * gridSize + z * gridSize * gridSize;
 
@@ -627,9 +627,9 @@ void buildHierarchy(Indexer* indexer, Node* node, shared_ptr<Buffer> points, int
 		int64_t iy = double(counterGridSize) * (y - min.y) / size.y;
 		int64_t iz = double(counterGridSize) * (z - min.z) / size.z;
 
-		ix = std::max(0ll, std::min(ix, counterGridSize - 1));
-		iy = std::max(0ll, std::min(iy, counterGridSize - 1));
-		iz = std::max(0ll, std::min(iz, counterGridSize - 1));
+		ix = std::max(int64_t(0), std::min(ix, counterGridSize - 1));
+		iy = std::max(int64_t(0), std::min(iy, counterGridSize - 1));
+		iz = std::max(int64_t(0), std::min(iz, counterGridSize - 1));
 
 		int64_t index = mortonEncode_magicbits(iz, iy, ix);