anydev/anyway
0
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

amazing cache #55

Merged
kscheidecker merged 22 commits from backend/grid-based-cache into main 2025-01-30 12:40:36 +00:00
Conversation 0 Commits 22 Files Changed 16 +33 -39
4 changed files with 33 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 1f4815c991 - Show all commits

6
backend/report.html
View File

@@ -328,7 +328,7 @@ div.media {
</head>
<body>
<h1 id="title">Backend Testing Report</h1>
<p>Report generated on 28-Jan-2025 at 14:07:50 by <a href="https://pypi.python.org/pypi/pytest-html">pytest-html</a>
<p>Report generated on 28-Jan-2025 at 16:30:56 by <a href="https://pypi.python.org/pypi/pytest-html">pytest-html</a>
v4.1.1</p>
<div id="environment-header">
<h2>Environment</h2>
@@ -382,7 +382,7 @@ div.media {
<h2>Summary</h2>
<div class="additional-summary prefix">
</div>
<p class="run-count">1 test took 159 ms.</p>
<p class="run-count">1 test took 00:00:10.</p>
<p class="filter">(Un)check the boxes to filter the results.</p>
<div class="summary__reload">
<div class="summary__reload__button hidden" onclick="location.reload()">
@@ -432,7 +432,7 @@ div.media {
</table>
</body>
<footer>
<div id="data-container" data-jsonblob="{&#34;environment&#34;: {&#34;Python&#34;: &#34;3.12.3&#34;, &#34;Platform&#34;: &#34;Linux-6.8.0-51-generic-x86_64-with-glibc2.39&#34;, &#34;Packages&#34;: {&#34;pytest&#34;: &#34;8.3.4&#34;, &#34;pluggy&#34;: &#34;1.5.0&#34;}, &#34;Plugins&#34;: {&#34;html&#34;: &#34;4.1.1&#34;, &#34;anyio&#34;: &#34;4.8.0&#34;, &#34;metadata&#34;: &#34;3.1.1&#34;}}, &#34;tests&#34;: {&#34;src/tests/test_main.py::test_turckheim&#34;: [{&#34;extras&#34;: [], &#34;result&#34;: &#34;Passed&#34;, &#34;testId&#34;: &#34;src/tests/test_main.py::test_turckheim&#34;, &#34;resultsTableRow&#34;: [&#34;&lt;td class=\&#34;col-result\&#34;&gt;Passed&lt;/td&gt;&#34;, &#34;&lt;td class=\&#34;col-testId\&#34;&gt;src/tests/test_main.py::test_turckheim&lt;/td&gt;&#34;, &#34;&lt;td&gt;start (0 | 0) - 6 - H\u00f4tel des Deux-Clefs (217 | 5) - 1 - H\u00f4tel de ville (238 | 5) - 6 - finish (0 | 0) - 0&lt;/td&gt;&#34;, &#34;&lt;td&gt;23 min&lt;/td&gt;&#34;, &#34;&lt;td&gt;20 min&lt;/td&gt;&#34;, &#34;&lt;td class=\&#34;col-duration\&#34;&gt;159 ms&lt;/td&gt;&#34;, &#34;&lt;td class=\&#34;col-links\&#34;&gt;&lt;/td&gt;&#34;], &#34;log&#34;: &#34;------------------------------ Captured log call -------------------------------\nDEBUG asyncio:selector_events.py:64 Using selector: EpollSelector\nINFO src.main:main.py:67 No end coordinates provided. Using start=end.\nDEBUG src.utils.landmarks_manager:landmarks_manager.py:76 Starting to fetch landmarks...\nDEBUG src.utils.landmarks_manager:landmarks_manager.py:88 Fetching sightseeing landmarks...\nINFO src.overpass.overpass:overpass.py:55 Cache hit.\nINFO src.overpass.overpass:overpass.py:55 Cache hit.\nINFO src.overpass.overpass:overpass.py:55 Cache hit.\nINFO src.overpass.overpass:overpass.py:55 Cache hit.\nINFO src.overpass.overpass:overpass.py:55 Cache hit.\nINFO src.overpass.overpass:overpass.py:55 Cache hit.\nDEBUG src.utils.landmarks_manager:landmarks_manager.py:204 Fetched 20 landmarks of type sightseeing in (48.079458619727674, 7.272726663391359, 48.08971738027232, 7.288083336608641)\nINFO src.utils.landmarks_manager:landmarks_manager.py:91 Found 20 sightseeing landmarks\nINFO src.overpass.overpass:overpass.py:55 Cache hit.\nINFO src.utils.cluster_manager:cluster_manager.py:145 Found 0 sightseeing clusters.\nDEBUG src.utils.landmarks_manager:landmarks_manager.py:100 Fetching nature landmarks...\nINFO src.overpass.overpass:overpass.py:55 Cache hit.\nINFO src.overpass.overpass:overpass.py:55 Cache hit.\nINFO src.overpass.overpass:overpass.py:55 Cache hit.\nINFO src.overpass.overpass:overpass.py:55 Cache hit.\nINFO src.overpass.overpass:overpass.py:55 Cache hit.\nINFO src.overpass.overpass:overpass.py:55 Cache hit.\nDEBUG src.utils.landmarks_manager:landmarks_manager.py:204 Fetched 24 landmarks of type nature in (48.079458619727674, 7.272726663391359, 48.08971738027232, 7.288083336608641)\nINFO src.utils.landmarks_manager:landmarks_manager.py:103 Found 24 nature landmarks\nINFO src.main:main.py:104 Fetched 25 landmarks in \t: 0.009 seconds\nDEBUG src.optimization.optimizer:optimizer.py:597 First results are out. Looking out for circles and correcting.\nINFO src.optimization.optimizer:optimizer.py:637 Re-optimized 0 times, objective value : 455\nDEBUG src.optimization.refiner:refiner.py:345 Using 5 minor landmarks around the predicted path\nDEBUG src.optimization.optimizer:optimizer.py:597 First results are out. Looking out for circles and correcting.\nINFO src.optimization.optimizer:optimizer.py:637 Re-optimized 0 times, objective value : 455\nDEBUG src.main:main.py:130 First stage optimization\t: 0.101 seconds\nDEBUG src.main:main.py:131 Second stage optimization\t: 0.033 seconds\nINFO src.main:main.py:132 Total computation time\t: 0.134 seconds\nINFO src.main:main.py:137 Generated a trip of 23 minutes with 4 landmarks in 0.143 seconds.\nINFO httpx:_client.py:1025 HTTP Request: POST http://testserver/trip/new &amp;quot;HTTP/1.1 200 OK&amp;quot;\n\n&#34;}]}, &#34;renderCollapsed&#34;: [&#34;passed&#34;], &#34;initialSort&#34;: &#34;result&#34;, &#34;title&#34;: &#34;Backend Testing Report&#34;}"></div>
<div id="data-container" data-jsonblob="{&#34;environment&#34;: {&#34;Python&#34;: &#34;3.12.3&#34;, &#34;Platform&#34;: &#34;Linux-6.8.0-51-generic-x86_64-with-glibc2.39&#34;, &#34;Packages&#34;: {&#34;pytest&#34;: &#34;8.3.4&#34;, &#34;pluggy&#34;: &#34;1.5.0&#34;}, &#34;Plugins&#34;: {&#34;html&#34;: &#34;4.1.1&#34;, &#34;anyio&#34;: &#34;4.8.0&#34;, &#34;metadata&#34;: &#34;3.1.1&#34;}}, &#34;tests&#34;: {&#34;src/tests/test_main.py::test_turckheim&#34;: [{&#34;extras&#34;: [], &#34;result&#34;: &#34;Passed&#34;, &#34;testId&#34;: &#34;src/tests/test_main.py::test_turckheim&#34;, &#34;resultsTableRow&#34;: [&#34;&lt;td class=\&#34;col-result\&#34;&gt;Passed&lt;/td&gt;&#34;, &#34;&lt;td class=\&#34;col-testId\&#34;&gt;src/tests/test_main.py::test_turckheim&lt;/td&gt;&#34;, &#34;&lt;td&gt;start (0 | 0) - 6 - H\u00f4tel des Deux-Clefs (217 | 5) - 1 - H\u00f4tel de ville (238 | 5) - 6 - finish (0 | 0) - 0&lt;/td&gt;&#34;, &#34;&lt;td&gt;23 min&lt;/td&gt;&#34;, &#34;&lt;td&gt;20 min&lt;/td&gt;&#34;, &#34;&lt;td class=\&#34;col-duration\&#34;&gt;00:00:10&lt;/td&gt;&#34;, &#34;&lt;td class=\&#34;col-links\&#34;&gt;&lt;/td&gt;&#34;], &#34;log&#34;: &#34;------------------------------ Captured log call -------------------------------\nDEBUG asyncio:selector_events.py:64 Using selector: EpollSelector\nINFO src.main:main.py:67 No end coordinates provided. Using start=end.\nDEBUG src.utils.landmarks_manager:landmarks_manager.py:76 Starting to fetch landmarks...\nDEBUG src.utils.landmarks_manager:landmarks_manager.py:88 Fetching sightseeing landmarks...\nINFO src.overpass.overpass:overpass.py:55 Cache hit for 1 quadrants.\nINFO src.overpass.overpass:overpass.py:55 Cache hit for 1 quadrants.\nINFO src.overpass.overpass:overpass.py:55 Cache hit for 1 quadrants.\nINFO src.overpass.overpass:overpass.py:59 Cache miss for 1 quadrants.\nINFO src.overpass.overpass:overpass.py:55 Cache hit for 1 quadrants.\nINFO src.overpass.overpass:overpass.py:55 Cache hit for 1 quadrants.\nDEBUG src.utils.landmarks_manager:landmarks_manager.py:204 Fetched 20 landmarks of type sightseeing in (48.079458619727674, 7.272726663391359, 48.08971738027232, 7.288083336608641)\nINFO src.utils.landmarks_manager:landmarks_manager.py:91 Found 20 sightseeing landmarks\nINFO src.overpass.overpass:overpass.py:59 Cache miss for 1 quadrants.\nINFO src.utils.cluster_manager:cluster_manager.py:145 Found 0 sightseeing clusters.\nDEBUG src.utils.landmarks_manager:landmarks_manager.py:100 Fetching nature landmarks...\nINFO src.overpass.overpass:overpass.py:59 Cache miss for 1 quadrants.\nINFO src.overpass.overpass:overpass.py:55 Cache hit for 1 quadrants.\nINFO src.overpass.overpass:overpass.py:55 Cache hit for 1 quadrants.\nINFO src.overpass.overpass:overpass.py:59 Cache miss for 1 quadrants.\nINFO src.overpass.overpass:overpass.py:59 Cache miss for 1 quadrants.\nINFO src.overpass.overpass:overpass.py:55 Cache hit for 1 quadrants.\nDEBUG src.utils.landmarks_manager:landmarks_manager.py:204 Fetched 10 landmarks of type nature in (48.079458619727674, 7.272726663391359, 48.08971738027232, 7.288083336608641)\nINFO src.utils.landmarks_manager:landmarks_manager.py:103 Found 10 nature landmarks\nINFO src.main:main.py:104 Fetched 22 landmarks in \t: 5.367 seconds\nDEBUG src.optimization.optimizer:optimizer.py:597 First results are out. Looking out for circles and correcting.\nINFO src.optimization.optimizer:optimizer.py:637 Re-optimized 0 times, objective value : 455\nDEBUG src.optimization.refiner:refiner.py:345 Using 5 minor landmarks around the predicted path\nDEBUG src.optimization.optimizer:optimizer.py:597 First results are out. Looking out for circles and correcting.\nINFO src.optimization.optimizer:optimizer.py:637 Re-optimized 0 times, objective value : 455\nDEBUG src.main:main.py:130 First stage optimization\t: 0.126 seconds\nDEBUG src.main:main.py:131 Second stage optimization\t: 0.033 seconds\nINFO src.main:main.py:132 Total computation time\t: 0.159 seconds\nINFO src.main:main.py:137 Generated a trip of 23 minutes with 4 landmarks in 5.526 seconds.\nDEBUG src.overpass.overpass:overpass.py:112 Cache set for b2fd047a07f9563c8c0925aad9d61052\nDEBUG src.overpass.overpass:overpass.py:112 Cache set for 7ee38297ba20e3bc47b984dce2785e22\nDEBUG src.overpass.overpass:overpass.py:112 Cache set for 7f98015c7bc3a9100e10bc1c1ddfa572\nDEBUG src.overpass.overpass:overpass.py:112 Cache set for 43424dbebba69d06a66a34728c3bb93e\nDEBUG src.overpass.overpass:overpass.py:112 Cache set for b7d4ee874adac132bc9e80d7172fab8e\nINFO httpx:_client.py:1025 HTTP Request: POST http://testserver/trip/new &amp;quot;HTTP/1.1 200 OK&amp;quot;\n\n&#34;}]}, &#34;renderCollapsed&#34;: [&#34;passed&#34;], &#34;initialSort&#34;: &#34;result&#34;, &#34;title&#34;: &#34;Backend Testing Report&#34;}"></div>
<script>
(function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i<t.length;i++)o(t[i]);return o}return r})()({1:[function(require,module,exports){
const { getCollapsedCategory, setCollapsedIds } = require('./storage.js')

2
backend/src/main.py
View File

@@ -14,7 +14,7 @@ from .utils.landmarks_manager import LandmarkManager
from .utils.toilets_manager import ToiletsManager
from .optimization.optimizer import Optimizer
from .optimization.refiner import Refiner
from .overpass.caching_strategy import fill_cache
from .overpass.overpass import fill_cache
from .cache import client as cache_client

33
backend/src/overpass/caching_strategy.py
View File

@@ -3,7 +3,6 @@ import json
import hashlib
from ..constants import OSM_CACHE_DIR, OSM_TYPES
from .overpass import Overpass
def get_cache_key(query: str) -> str:
@@ -13,6 +12,7 @@ def get_cache_key(query: str) -> str:
"""
return hashlib.md5(query.encode('utf-8')).hexdigest()
class CachingStrategyBase:
"""
Base class for implementing caching strategies.
@@ -25,15 +25,10 @@ class CachingStrategyBase:
"""Store data in the cache with the specified key."""
raise NotImplementedError('Subclass should implement set')
def set_hollow(self, key, cell: tuple, osm_types: OSM_TYPES,
selector: str, conditions=[], out='center'):
def set_hollow(self, key, **kwargs):
"""Create a hollow (empty) cache entry with a specific key."""
raise NotImplementedError('Subclass should implement set_hollow')
def fill_hollow(self, key, value):
"""Fill in the cache for an existing hollow entry."""
raise NotImplementedError('Subclass should implement fill_hollow')
def close(self):
"""Clean up or close any resources used by the caching strategy."""
@@ -134,27 +129,3 @@ class CachingStrategy:
selector: str, conditions: list=None, out='center'):
"""Create a hollow cache entry."""
cls.__strategy.set_hollow(key, cell, osm_types, selector, conditions, out)
def fill_cache():
"""
Scans the specified cache directory for files starting with 'hollow_' and attempts to load
their contents as JSON to fill the cache of the Overpass system.
"""
overpass = Overpass()
with os.scandir(OSM_CACHE_DIR) as it:
for entry in it:
if entry.is_file() and entry.name.startswith('hollow_'):
try :
# Read the whole file content as a string
with open(entry.path, 'r') as f:
# load data and fill the cache with the query and key
json_data = json.load(f)
overpass.fill_cache(json_data)
# Now delete the file as the cache is filled
os.remove(entry.path)
except Exception as exc :
overpass.logger.error(f'An error occured while parsing file {f} as .json file')

31
backend/src/overpass/overpass.py
View File

@@ -52,10 +52,11 @@ class Overpass :
# If there is no missing data, return the cached responses
if not hollow_cache_keys :
self.logger.info('Cache hit.')
self.logger.info(f'Cache hit for {len(cached_responses)} quadrants.')
return self._combine_cached_data(cached_responses)
# TODO If there is SOME missing data : hybrid stuff with partial cache
self.logger.info(f'Cache miss for {len(hollow_cache_keys)} quadrants.')
# Missing data: Make a query to Overpass API
query_str = Overpass.build_query(bbox, osm_types, selector, conditions, out)
@@ -80,9 +81,7 @@ class Overpass :
response_data = response.read().decode('utf-8') # Convert the HTTPResponse to a string
data = json.loads(response_data) # Load the JSON from the string
elements = data.get('elements', [])
self.logger.info(f'Cache miss. Fetching data through Overpass.')
self.logger.debug(f'Query = {query_str}')
# self.logger.debug(f'Query = {query_str}')
return elements
except urllib.error.URLError as e:
@@ -321,3 +320,27 @@ def get_base_info(elem: dict, osm_type: OSM_TYPES, with_name=False) :
return osm_id, coords, name
else :
return osm_id, coords
def fill_cache():
"""
Scans the specified cache directory for files starting with 'hollow_' and attempts to load
their contents as JSON to fill the cache of the Overpass system.
"""
overpass = Overpass()
with os.scandir(OSM_CACHE_DIR) as it:
for entry in it:
if entry.is_file() and entry.name.startswith('hollow_'):
try :
# Read the whole file content as a string
with open(entry.path, 'r') as f:
# load data and fill the cache with the query and key
json_data = json.load(f)
overpass.fill_cache(json_data)
# Now delete the file as the cache is filled
os.remove(entry.path)
except Exception as exc :
overpass.logger.error(f'An error occured while parsing file {entry.path} as .json file')