wip

src/client/mapblock_mesh.cpp

@@ -778,7 +778,7 @@ MapBlockMesh::MapBlockMesh(Client *client, MeshMakeData *data):
 							r /= color_far_scale;
 							g /= color_far_scale;
 							b /= color_far_scale;
-							const video::SColor c(255 / fscale, r, g, b);
+							const video::SColor c(0 / fscale, r, g, b);
 							const auto lpos_rel = lp.first - bpos_rel;
 							const auto coord = posToFloat(lpos_rel, BS);
 							video::S3DVertex v(coord.X, coord.Y, coord.Z, 0.0f, 0.0f,

src/mapgen/earth/arnis_adapter.cpp

@@ -6,4 +6,5 @@ XZ::operator XZPoint()
 {
 	return XZPoint{X, Y};
 }
+const std::vector<ProcessedNode> ProcessedElement::dummy_nodes;
 }

src/mapgen/earth/arnis_adapter.h

@@ -172,10 +172,12 @@ public:
 		}
 	}
 
+	const static std::vector<ProcessedNode> dummy_nodes;
 	const std::vector<ProcessedNode> &nodes() const
 	{
 		if (is_way())
 			return as_way().nodes;
+		return dummy_nodes;
 	}
 
 	static ProcessedElement FromNode(const ProcessedNode &n)

src/mapgen/mapgen_earth.cpp

@@ -29,6 +29,7 @@ along with Freeminer.  If not, see <http://www.gnu.org/licenses/>.
 #include <system_error>
 #include <tuple>
 
+#include "constants.h"
 #include "debug/dump.h"
 #include "emerge.h"
 #include "filesys.h"
@@ -47,6 +48,7 @@ along with Freeminer.  If not, see <http://www.gnu.org/licenses/>.
 #include "nodedef.h"
 #include "serverenvironment.h"
 #include "settings.h"
+#include "util/timetaker.h"
 #include "voxel.h"
 #include "voxelalgorithms.h"
 #if USE_OSMIUM
@@ -65,6 +67,7 @@ along with Freeminer.  If not, see <http://www.gnu.org/licenses/>.
 #include <osmium/osm/way.hpp>
 #include <osmium/tags/tags_filter.hpp>
 #endif
+
 std::unique_ptr<maps_holder_t> MapgenEarth::maps_holder;
 
 void MapgenEarthParams::setDefaultSettings(Settings *settings)
@@ -138,7 +141,7 @@ MapgenEarth::MapgenEarth(MapgenEarthParams *params_, EmergeParams *emerge) :
 	// mg_params = (MapgenEarthParams *)params_->sparams;
 	mg_params = params_;
 
-	Json::Value &params = mg_params->params;
+	const Json::Value &params = mg_params->params;
 	flags = 0;
 
 	if (params.get("light", 0).asBool())
@@ -216,8 +219,9 @@ MapgenEarth::~MapgenEarth()
 
 MapNode MapgenEarth::layers_get(float value, float max)
 {
-	auto layer_index = rangelim((unsigned int)myround((value / max) * layers_node_size),
-			0, layers_node_size - 1);
+	const auto layer_index =
+			rangelim((unsigned int)myround((value / max) * layers_node_size), 0,
+					layers_node_size - 1);
 	return layers_node[layer_index];
 }
 
@@ -273,7 +277,7 @@ v2pos_t MapgenEarth::ll_to_pos(const ll &l)
 pos_t MapgenEarth::get_height(pos_t x, pos_t z)
 {
 	const auto tc = pos_to_ll(x, z);
-	auto y = maps_holder->hgt_reader.get(tc.lat, tc.lon);
+	const auto y = maps_holder->hgt_reader.get(tc.lat, tc.lon);
 	return ceil(y / scale.Y) - center.Y;
 }
 
@@ -289,18 +293,18 @@ int MapgenEarth::getGroundLevelAtPoint(v2pos_t p)
 
 int MapgenEarth::generateTerrain()
 {
-	MapNode n_ice(c_ice);
+	const MapNode n_ice(c_ice);
 	u32 index = 0;
-	auto em = vm->m_area.getExtent();
+	const auto em = vm->m_area.getExtent();
 
 	for (pos_t z = node_min.Z; z <= node_max.Z; z++) {
 		for (pos_t x = node_min.X; x <= node_max.X; x++, index++) {
-			auto heat =
+			const auto heat =
 					m_emerge->env->m_use_weather
 							? m_emerge->env->getServerMap().updateBlockHeat(m_emerge->env,
 									  v3pos_t(x, node_max.Y, z), nullptr, &heat_cache)
 							: 0;
-			auto height = get_height(x, z);
+			const auto height = get_height(x, z);
 			u32 i = vm->m_area.index(x, node_min.Y, z);
 			for (pos_t y = node_min.Y; y <= node_max.Y; y++) {
 				bool underground = height >= y;
@@ -375,7 +379,7 @@ void MapgenEarth::generateBuildings()
 {
 
 #if USE_OSMIUM
-	TimeTaker timer("earth buildings");
+	TimeTaker timer("earth buildings", {}, PRECISION_MILLI);
 	std::string use_file;
 	try {
 
@@ -396,7 +400,7 @@ void MapgenEarth::generateBuildings()
 		std::snprintf(buff, sizeof(buff), "%c%02d%c%03d-%s.osm.pbf",
 				lat_dec >= 0 ? 'N' : 'S', abs(lat_dec), lon_dec > 0 ? 'W' : 'E',
 				abs(lon_dec), timestamp.c_str());
-		std::string filename = buff;
+		const std::string filename = buff;
 		const auto base_full_name = folder + DIR_DELIM + filename;
 		if (!std::filesystem::exists(base_full_name)) {
 			const auto lock = std::lock_guard(maps_holder->osm_http_lock);
@@ -421,8 +425,10 @@ void MapgenEarth::generateBuildings()
 
 				std::stringstream cmd;
 				// TODO: use osmium tool as lib
-				cmd << "osmium extract --output-format pbf --strategy smart " << "--bbox "
-					<< bbox << " --output " << filename << ".tmp" << " " << path_name;
+				// --option types=multipolygon,route
+				cmd << "osmium extract --output-format pbf --strategy smart --option types=any "
+					<< "--bbox " << bbox << " --output " << filename << ".tmp" << " "
+					<< path_name;
 				exec_to_string(cmd.str());
 				if (!std::filesystem::exists(filename + ".tmp")) {
 					return false;
@@ -461,35 +467,42 @@ void MapgenEarth::generateBuildings()
 				if (!try_extract(use_file, bbox_next, filename_next)) {
 					break;
 				}
+
 				if (div >= 1000) {
 					maps_holder->files_to_delete.emplace_back(filename_next);
 				}
+
 				use_file = filename_next;
 				bbox = bbox_next;
 			}
 		}
 
 		if (std::filesystem::exists(use_file) && std::filesystem::file_size(use_file)) {
+			const auto lock = std::lock_guard{maps_holder->osm_bbox_lock};
 			if (!maps_holder->osm_bbox.contains(bbox)) {
 				const auto osm = std::make_shared<hdl>(this, use_file);
-				const auto lock = maps_holder->osm_bbox.lock_unique_rec();
+				//const auto lock = maps_holder->osm_bbox.lock_unique_rec();
 				if (!maps_holder->osm_bbox.contains(bbox)) {
 					maps_holder->osm_bbox.emplace(bbox, osm);
 				}
 			}
 		}
 
-		if (const auto &hdlr = maps_holder->osm_bbox.get(bbox)) {
-			hdlr->apply(this);
+		{
+			const auto lock = std::lock_guard{maps_holder->osm_bbox_lock};
+			if (const auto &hdlr = maps_holder->osm_bbox.get(bbox)) {
+				hdlr.value()->apply(this);
+			}
 		}
 	} catch (const std::exception &ex) {
 		warningstream << node_min << " : " << ex.what() << " file=" << use_file << "\n";
 	}
 
-	verbosestream << "Buildings stat: " << node_min << " .. " << node_max
+	verbosestream << "Buildings stat: " << node_min << " .. " << node_max << " "
 				  << " set=" << stat.set << " miss=" << stat.miss
 				  << " level=" << stat.level << " check=" << stat.check
-				  << " fill=" << stat.fill << " per=" << timer.getTimerTime() << "\n";
+				  << " fill=" << stat.fill << " per=" << timer.getTimerTime()
+				  << " mh=" << maps_holder->osm_bbox.size() << "\n";
 	stat.clean();
 #endif
 }
@@ -514,7 +527,9 @@ weather::humidity_t MapgenEarth::calcBlockHumidity(const v3pos_t &p, uint64_t se
 maps_holder_t::~maps_holder_t()
 {
 	for (const auto &file : files_to_delete) {
+#if NDEBUG
 		std::error_code ec;
 		std::filesystem::remove(file, ec);
+#endif
 	}
 };

src/mapgen/mapgen_earth.h

@@ -32,6 +32,7 @@ along with Freeminer.  If not, see <http://www.gnu.org/licenses/>.
 #include "filesys.h"
 #include "threading/concurrent_map.h"
 #include "threading/concurrent_vector.h"
+#include "util/lrucache.hpp"
 
 typedef core::vector2d<double> v2d;
 
@@ -73,8 +74,8 @@ struct maps_holder_t
 {
 	hgts hgt_reader{porting::path_cache + DIR_DELIM + "earth"};
 	using osm_ptr = std::shared_ptr<handler_i>;
-	concurrent_shared_map<std::string, osm_ptr> osm_by_path;
-	concurrent_shared_map<std::string, osm_ptr> osm_bbox;
+	lru_cache<std::string, osm_ptr, 50> osm_bbox;
+	std::mutex osm_bbox_lock;
 	std::mutex osm_http_lock;
 	std::mutex osm_extract_lock;
 	concurrent_shared_vector<std::string> files_to_delete;

src/util/lrucache.hpp

@@ -0,0 +1,279 @@
+// https://github.com/Guiorgy/cpp-lru-cache/blob/std-only/include/lrucache.hpp
+/*
+** File: lrucache.hpp
+**
+** Author: Alexander Ponomarev
+** Created on June 20, 2013, 5:09 PM
+**
+** Author: Guiorgy
+** Forked on October 23, 2024
+*/
+
+#pragma once
+
+#include <unordered_map>
+#include <functional>
+#include <optional>
+#include <utility>
+#include <tuple>
+#include <list>
+
+// A helper macro that determines whether a specific attribute is supported by the current compiler. If __has_cpp_attribute is not defined, __cplusplus alone is used as a fallback.
+#ifdef __has_cpp_attribute
+	#define GUIORGY_ATTRIBUTE_AVAILABLE(attribute_token, test_value, cpp_version) \
+		(__has_cpp_attribute(attribute_token) >= test_value && __cplusplus >= cpp_version)
+#else
+	#define GUIORGY_ATTRIBUTE_AVAILABLE(attribute_token, test_value, cpp_version) \
+		(__cplusplus >= cpp_version)
+#endif
+
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunknown-warning-option"
+#pragma clang diagnostic ignored "-Wreserved-attribute-identifier"
+#endif
+#if !GUIORGY_ATTRIBUTE_AVAILABLE(nodiscard, 201907L, 202002L)
+	#ifdef nodiscard
+		#define GUIORGY_nodiscard_BEFORE
+		#undef nodiscard
+	#endif
+	#define nodiscard(explanation) nodiscard
+#endif
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
+
+template<typename key_t, typename value_t, const std::size_t max_size>
+class lru_cache final {
+	static_assert(max_size > 0);
+
+	typedef typename std::pair<key_t, value_t> key_value_pair_t;
+	typedef typename std::list<key_value_pair_t>::iterator list_iterator_t;
+	typedef typename std::list<key_value_pair_t>::reverse_iterator list_reverse_iterator_t;
+	typedef typename std::unordered_map<key_t, list_iterator_t>::iterator map_iterator_t;
+
+public:
+	typedef typename std::list<key_value_pair_t>::const_iterator iterator;
+	typedef typename std::list<key_value_pair_t>::const_iterator const_iterator;
+	typedef typename std::list<key_value_pair_t>::const_reverse_iterator reverse_iterator;
+	typedef typename std::list<key_value_pair_t>::const_reverse_iterator const_reverse_iterator;
+
+private:
+	std::list<key_value_pair_t> _cache_items_list;
+	std::unordered_map<key_t, list_iterator_t> _cache_items_map;
+
+public:
+	lru_cache() = default;
+	~lru_cache() = default;
+	lru_cache(const lru_cache& other) {
+		_cache_items_list = other._cache_items_list;
+
+		_cache_items_map.reserve(_cache_items_list.size());
+		for (list_iterator_t it = _cache_items_list.begin(); it != _cache_items_list.end(); ++it) {
+			_cache_items_map.emplace_hint(_cache_items_map.end(), it->first, it);
+		}
+	}
+	lru_cache(lru_cache&&) = default;
+	lru_cache& operator=(lru_cache const& other) {
+		_cache_items_list = other._cache_items_list;
+
+		_cache_items_map.reserve(_cache_items_list.size());
+		for (list_iterator_t it = _cache_items_list.begin(); it != _cache_items_list.end(); ++it) {
+			_cache_items_map.emplace_hint(_cache_items_map.end(), it->first, it);
+		}
+
+		return *this;
+	}
+	lru_cache& operator=(lru_cache &&) = default;
+
+private:
+	void put(const key_t& key) {
+		map_iterator_t it = _cache_items_map.find(key);
+
+		if (it != _cache_items_map.end()) {
+			_cache_items_list.erase(it->second);
+			it->second = _cache_items_list.begin();
+		} else {
+			_cache_items_map.emplace_hint(it, key, _cache_items_list.begin());
+		}
+
+		if (_cache_items_map.size() > max_size) {
+			list_reverse_iterator_t last = _cache_items_list.rbegin();
+			_cache_items_map.erase(last->first);
+			_cache_items_list.pop_back();
+		}
+	}
+
+public:
+	void put(const key_t& key, const value_t& value) {
+		_cache_items_list.emplace_front(key, value);
+		put(key);
+	}
+
+	void put(const key_t& key, value_t&& value) {
+		_cache_items_list.emplace_front(key, std::move(value));
+		put(key);
+	}
+
+	template<typename... ValueArgs>
+	const value_t& emplace(const key_t& key, ValueArgs&&... value_args) {
+		const value_t& value = _cache_items_list.emplace_front(
+			std::piecewise_construct,
+			std::forward_as_tuple(key),
+			std::forward_as_tuple(std::forward<ValueArgs>(value_args)...)
+		).second;
+		put(key);
+		return value;
+	}
+
+	[[nodiscard("Use touch(const key_t& key) instead if all you want is to push the key to the bottom of the removal order")]] const std::optional<value_t> get(const key_t& key) {
+		map_iterator_t it = _cache_items_map.find(key);
+
+		if (it == _cache_items_map.end()) {
+			return std::nullopt;
+		} else {
+			if (it->second != _cache_items_list.begin()) {
+				_cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+			}
+			return it->second->second;
+		}
+	}
+
+	[[nodiscard("Use touch(const key_t& key) instead if all you want is to push the key to the bottom of the removal order")]] const std::optional<std::reference_wrapper<const value_t>> get_ref(const key_t& key) {
+		map_iterator_t it = _cache_items_map.find(key);
+
+		if (it == _cache_items_map.end()) {
+			return std::nullopt;
+		} else {
+			if (it->second != _cache_items_list.begin()) {
+				_cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+			}
+			return std::make_optional(std::cref(it->second->second));
+		}
+	}
+
+	[[nodiscard("Use touch(const key_t& key) instead if all you want is to push the key to the bottom of the removal order")]] bool try_get(const key_t& key, value_t& value_out) {
+		map_iterator_t it = _cache_items_map.find(key);
+
+		if (it == _cache_items_map.end()) {
+			return false;
+		} else {
+			if (it->second != _cache_items_list.begin()) {
+				_cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+			}
+			value_out = it->second->second;
+			return true;
+		}
+	}
+
+	[[nodiscard("Use touch(const key_t& key) instead if all you want is to push the key to the bottom of the removal order")]] bool try_get_ref(const key_t& key, const value_t*& value_out) {
+		map_iterator_t it = _cache_items_map.find(key);
+
+		if (it == _cache_items_map.end()) {
+			value_out = nullptr;
+			return false;
+		} else {
+			if (it->second != _cache_items_list.begin()) {
+				_cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+			}
+			value_out = &(it->second->second);
+			return true;
+		}
+	}
+
+	bool touch(const key_t& key) {
+		map_iterator_t it = _cache_items_map.find(key);
+
+		if (it == _cache_items_map.end()) {
+			return false;
+		} else {
+			if (it->second != _cache_items_list.begin()) {
+				_cache_items_list.splice(_cache_items_list.begin(), _cache_items_list, it->second);
+			}
+			return true;
+		}
+	}
+
+	bool remove(const key_t& key) {
+		map_iterator_t it = _cache_items_map.find(key);
+
+		if (it != _cache_items_map.end()) {
+			_cache_items_list.erase(it->second);
+			_cache_items_map.erase(it);
+			return true;
+		} else {
+			return false;
+		}
+	}
+
+	[[nodiscard]] bool exists(const key_t& key) const {
+		return _cache_items_map.find(key) != _cache_items_map.end();
+	}
+
+    [[nodiscard]] bool contains(const key_t& key) const {
+		return _cache_items_map.find(key) != _cache_items_map.end();
+	}
+
+	[[nodiscard]] std::size_t size() const noexcept {
+		return _cache_items_map.size();
+	}
+
+	void clear() noexcept {
+		_cache_items_map.clear();
+		_cache_items_list.clear();
+	}
+
+	[[nodiscard]] const_iterator cbegin() const noexcept {
+		return _cache_items_list.cbegin();
+	}
+
+	[[nodiscard]] const_iterator begin() const noexcept {
+		return cbegin();
+	}
+
+	[[nodiscard]] const_iterator cend() const noexcept {
+		return _cache_items_list.cend();
+	}
+
+	[[nodiscard]] const_iterator end() const noexcept {
+		return cend();
+	}
+
+	[[nodiscard]] const_reverse_iterator crbegin() const noexcept {
+		return _cache_items_list.crbegin();
+	}
+
+	[[nodiscard]] const_iterator rbegin() const noexcept {
+		return crbegin();
+	}
+
+	[[nodiscard]] const_reverse_iterator crend() const noexcept {
+		return _cache_items_list.crend();
+	}
+
+	[[nodiscard]] const_iterator rend() const noexcept {
+		return crend();
+	}
+};
+
+// Restore nodiscard if it was already defined, otherwise undefine it
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunknown-warning-option"
+#pragma clang diagnostic ignored "-Wreserved-attribute-identifier"
+#endif
+#ifdef GUIORGY_nodiscard_BEFORE
+	#undef nodiscard
+	#define nodiscard GUIORGY_nodiscard_BEFORE
+	#undef GUIORGY_nodiscard_BEFORE
+#else
+	#ifdef nodiscard
+		#undef nodiscard
+	#endif
+#endif
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
+
+// Cleanup of internal macros.
+#undef GUIORGY_ATTRIBUTE_AVAILABLE

util/autotest/auto.pl

@@ -414,7 +414,7 @@ $commands = {
         file_append(
             "$config->{logdir}/run.sh",
             join "\n",
-            qq{# } . join(' ', $0, map { /[\s"]/ ? "'" . $_ . "'" : $_ } @ARGV),
+            qq{# } . join(' ', $0, map { /[()\s"]/ ? "'" . $_ . "'" : $_ } @ARGV),
             qq{cd "$build_dir"},
             ""
         );