From 1f4815c991b43dde0a1c6c025f03f600c2243da7 Mon Sep 17 00:00:00 2001
From: Helldragon67 <kilian.scheidecker@orange.fr>
Date: Tue, 28 Jan 2025 16:31:15 +0100
Subject: [PATCH] better logs

---
 backend/report.html                      |  6 ++---
 backend/src/main.py                      |  2 +-
 backend/src/overpass/caching_strategy.py | 33 ++----------------------
 backend/src/overpass/overpass.py         | 31 +++++++++++++++++++---
 4 files changed, 33 insertions(+), 39 deletions(-)
diff --git a/backend/report.html b/backend/report.html
index a74ab9d..44e772a 100644
--- a/backend/report.html
+++ b/backend/report.html
@@ -328,7 +328,7 @@ div.media {
   </head>
   <body>
     <h1 id="title">Backend Testing Report</h1>
-    <p>Report generated on 28-Jan-2025 at 14:07:50 by <a href="https://pypi.python.org/pypi/pytest-html">pytest-html</a>
+    <p>Report generated on 28-Jan-2025 at 16:30:56 by <a href="https://pypi.python.org/pypi/pytest-html">pytest-html</a>
         v4.1.1</p>
     <div id="environment-header">
       <h2>Environment</h2>
@@ -382,7 +382,7 @@ div.media {
         <h2>Summary</h2>
         <div class="additional-summary prefix">
         </div>
-        <p class="run-count">1 test took 159 ms.</p>
+        <p class="run-count">1 test took 00:00:10.</p>
         <p class="filter">(Un)check the boxes to filter the results.</p>
         <div class="summary__reload">
           <div class="summary__reload__button hidden" onclick="location.reload()">
@@ -432,7 +432,7 @@ div.media {
     </table>
   </body>
   <footer>
-    <div id="data-container" data-jsonblob="{&#34;environment&#34;: {&#34;Python&#34;: &#34;3.12.3&#34;, &#34;Platform&#34;: &#34;Linux-6.8.0-51-generic-x86_64-with-glibc2.39&#34;, &#34;Packages&#34;: {&#34;pytest&#34;: &#34;8.3.4&#34;, &#34;pluggy&#34;: &#34;1.5.0&#34;}, &#34;Plugins&#34;: {&#34;html&#34;: &#34;4.1.1&#34;, &#34;anyio&#34;: &#34;4.8.0&#34;, &#34;metadata&#34;: &#34;3.1.1&#34;}}, &#34;tests&#34;: {&#34;src/tests/test_main.py::test_turckheim&#34;: [{&#34;extras&#34;: [], &#34;result&#34;: &#34;Passed&#34;, &#34;testId&#34;: &#34;src/tests/test_main.py::test_turckheim&#34;, &#34;resultsTableRow&#34;: [&#34;&lt;td class=\&#34;col-result\&#34;&gt;Passed&lt;/td&gt;&#34;, &#34;&lt;td class=\&#34;col-testId\&#34;&gt;src/tests/test_main.py::test_turckheim&lt;/td&gt;&#34;, &#34;&lt;td&gt;start (0 | 0) - 6 - H\u00f4tel des Deux-Clefs (217 | 5) - 1 - H\u00f4tel de ville (238 | 5) - 6 - finish (0 | 0) - 0&lt;/td&gt;&#34;, &#34;&lt;td&gt;23 min&lt;/td&gt;&#34;, &#34;&lt;td&gt;20 min&lt;/td&gt;&#34;, &#34;&lt;td class=\&#34;col-duration\&#34;&gt;159 ms&lt;/td&gt;&#34;, &#34;&lt;td class=\&#34;col-links\&#34;&gt;&lt;/td&gt;&#34;], &#34;log&#34;: &#34;------------------------------ Captured log call -------------------------------\nDEBUG    asyncio:selector_events.py:64 Using selector: EpollSelector\nINFO     src.main:main.py:67 No end coordinates provided. Using start=end.\nDEBUG    src.utils.landmarks_manager:landmarks_manager.py:76 Starting to fetch landmarks...\nDEBUG    src.utils.landmarks_manager:landmarks_manager.py:88 Fetching sightseeing landmarks...\nINFO     src.overpass.overpass:overpass.py:55 Cache hit.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit.\nDEBUG    src.utils.landmarks_manager:landmarks_manager.py:204 Fetched 20 landmarks of type sightseeing in (48.079458619727674, 7.272726663391359, 48.08971738027232, 7.288083336608641)\nINFO     src.utils.landmarks_manager:landmarks_manager.py:91 Found 20 sightseeing landmarks\nINFO     src.overpass.overpass:overpass.py:55 Cache hit.\nINFO     src.utils.cluster_manager:cluster_manager.py:145 Found 0 sightseeing clusters.\nDEBUG    src.utils.landmarks_manager:landmarks_manager.py:100 Fetching nature landmarks...\nINFO     src.overpass.overpass:overpass.py:55 Cache hit.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit.\nDEBUG    src.utils.landmarks_manager:landmarks_manager.py:204 Fetched 24 landmarks of type nature in (48.079458619727674, 7.272726663391359, 48.08971738027232, 7.288083336608641)\nINFO     src.utils.landmarks_manager:landmarks_manager.py:103 Found 24 nature landmarks\nINFO     src.main:main.py:104 Fetched 25 landmarks in  \t: 0.009 seconds\nDEBUG    src.optimization.optimizer:optimizer.py:597 First results are out. Looking out for circles and correcting.\nINFO     src.optimization.optimizer:optimizer.py:637 Re-optimized 0 times, objective value : 455\nDEBUG    src.optimization.refiner:refiner.py:345 Using 5 minor landmarks around the predicted path\nDEBUG    src.optimization.optimizer:optimizer.py:597 First results are out. Looking out for circles and correcting.\nINFO     src.optimization.optimizer:optimizer.py:637 Re-optimized 0 times, objective value : 455\nDEBUG    src.main:main.py:130 First stage optimization\t: 0.101 seconds\nDEBUG    src.main:main.py:131 Second stage optimization\t: 0.033 seconds\nINFO     src.main:main.py:132 Total computation time\t: 0.134 seconds\nINFO     src.main:main.py:137 Generated a trip of 23 minutes with 4 landmarks in 0.143 seconds.\nINFO     httpx:_client.py:1025 HTTP Request: POST http://testserver/trip/new &amp;quot;HTTP/1.1 200 OK&amp;quot;\n\n&#34;}]}, &#34;renderCollapsed&#34;: [&#34;passed&#34;], &#34;initialSort&#34;: &#34;result&#34;, &#34;title&#34;: &#34;Backend Testing Report&#34;}"></div>
+    <div id="data-container" data-jsonblob="{&#34;environment&#34;: {&#34;Python&#34;: &#34;3.12.3&#34;, &#34;Platform&#34;: &#34;Linux-6.8.0-51-generic-x86_64-with-glibc2.39&#34;, &#34;Packages&#34;: {&#34;pytest&#34;: &#34;8.3.4&#34;, &#34;pluggy&#34;: &#34;1.5.0&#34;}, &#34;Plugins&#34;: {&#34;html&#34;: &#34;4.1.1&#34;, &#34;anyio&#34;: &#34;4.8.0&#34;, &#34;metadata&#34;: &#34;3.1.1&#34;}}, &#34;tests&#34;: {&#34;src/tests/test_main.py::test_turckheim&#34;: [{&#34;extras&#34;: [], &#34;result&#34;: &#34;Passed&#34;, &#34;testId&#34;: &#34;src/tests/test_main.py::test_turckheim&#34;, &#34;resultsTableRow&#34;: [&#34;&lt;td class=\&#34;col-result\&#34;&gt;Passed&lt;/td&gt;&#34;, &#34;&lt;td class=\&#34;col-testId\&#34;&gt;src/tests/test_main.py::test_turckheim&lt;/td&gt;&#34;, &#34;&lt;td&gt;start (0 | 0) - 6 - H\u00f4tel des Deux-Clefs (217 | 5) - 1 - H\u00f4tel de ville (238 | 5) - 6 - finish (0 | 0) - 0&lt;/td&gt;&#34;, &#34;&lt;td&gt;23 min&lt;/td&gt;&#34;, &#34;&lt;td&gt;20 min&lt;/td&gt;&#34;, &#34;&lt;td class=\&#34;col-duration\&#34;&gt;00:00:10&lt;/td&gt;&#34;, &#34;&lt;td class=\&#34;col-links\&#34;&gt;&lt;/td&gt;&#34;], &#34;log&#34;: &#34;------------------------------ Captured log call -------------------------------\nDEBUG    asyncio:selector_events.py:64 Using selector: EpollSelector\nINFO     src.main:main.py:67 No end coordinates provided. Using start=end.\nDEBUG    src.utils.landmarks_manager:landmarks_manager.py:76 Starting to fetch landmarks...\nDEBUG    src.utils.landmarks_manager:landmarks_manager.py:88 Fetching sightseeing landmarks...\nINFO     src.overpass.overpass:overpass.py:55 Cache hit for 1 quadrants.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit for 1 quadrants.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit for 1 quadrants.\nINFO     src.overpass.overpass:overpass.py:59 Cache miss for 1 quadrants.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit for 1 quadrants.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit for 1 quadrants.\nDEBUG    src.utils.landmarks_manager:landmarks_manager.py:204 Fetched 20 landmarks of type sightseeing in (48.079458619727674, 7.272726663391359, 48.08971738027232, 7.288083336608641)\nINFO     src.utils.landmarks_manager:landmarks_manager.py:91 Found 20 sightseeing landmarks\nINFO     src.overpass.overpass:overpass.py:59 Cache miss for 1 quadrants.\nINFO     src.utils.cluster_manager:cluster_manager.py:145 Found 0 sightseeing clusters.\nDEBUG    src.utils.landmarks_manager:landmarks_manager.py:100 Fetching nature landmarks...\nINFO     src.overpass.overpass:overpass.py:59 Cache miss for 1 quadrants.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit for 1 quadrants.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit for 1 quadrants.\nINFO     src.overpass.overpass:overpass.py:59 Cache miss for 1 quadrants.\nINFO     src.overpass.overpass:overpass.py:59 Cache miss for 1 quadrants.\nINFO     src.overpass.overpass:overpass.py:55 Cache hit for 1 quadrants.\nDEBUG    src.utils.landmarks_manager:landmarks_manager.py:204 Fetched 10 landmarks of type nature in (48.079458619727674, 7.272726663391359, 48.08971738027232, 7.288083336608641)\nINFO     src.utils.landmarks_manager:landmarks_manager.py:103 Found 10 nature landmarks\nINFO     src.main:main.py:104 Fetched 22 landmarks in  \t: 5.367 seconds\nDEBUG    src.optimization.optimizer:optimizer.py:597 First results are out. Looking out for circles and correcting.\nINFO     src.optimization.optimizer:optimizer.py:637 Re-optimized 0 times, objective value : 455\nDEBUG    src.optimization.refiner:refiner.py:345 Using 5 minor landmarks around the predicted path\nDEBUG    src.optimization.optimizer:optimizer.py:597 First results are out. Looking out for circles and correcting.\nINFO     src.optimization.optimizer:optimizer.py:637 Re-optimized 0 times, objective value : 455\nDEBUG    src.main:main.py:130 First stage optimization\t: 0.126 seconds\nDEBUG    src.main:main.py:131 Second stage optimization\t: 0.033 seconds\nINFO     src.main:main.py:132 Total computation time\t: 0.159 seconds\nINFO     src.main:main.py:137 Generated a trip of 23 minutes with 4 landmarks in 5.526 seconds.\nDEBUG    src.overpass.overpass:overpass.py:112 Cache set for b2fd047a07f9563c8c0925aad9d61052\nDEBUG    src.overpass.overpass:overpass.py:112 Cache set for 7ee38297ba20e3bc47b984dce2785e22\nDEBUG    src.overpass.overpass:overpass.py:112 Cache set for 7f98015c7bc3a9100e10bc1c1ddfa572\nDEBUG    src.overpass.overpass:overpass.py:112 Cache set for 43424dbebba69d06a66a34728c3bb93e\nDEBUG    src.overpass.overpass:overpass.py:112 Cache set for b7d4ee874adac132bc9e80d7172fab8e\nINFO     httpx:_client.py:1025 HTTP Request: POST http://testserver/trip/new &amp;quot;HTTP/1.1 200 OK&amp;quot;\n\n&#34;}]}, &#34;renderCollapsed&#34;: [&#34;passed&#34;], &#34;initialSort&#34;: &#34;result&#34;, &#34;title&#34;: &#34;Backend Testing Report&#34;}"></div>
     <script>
       (function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i<t.length;i++)o(t[i]);return o}return r})()({1:[function(require,module,exports){
 const { getCollapsedCategory, setCollapsedIds } = require('./storage.js')
diff --git a/backend/src/main.py b/backend/src/main.py
index 54c8b44..104ff08 100644
--- a/backend/src/main.py
+++ b/backend/src/main.py
@@ -14,7 +14,7 @@ from .utils.landmarks_manager import LandmarkManager
 from .utils.toilets_manager import ToiletsManager
 from .optimization.optimizer import Optimizer
 from .optimization.refiner import Refiner
-from .overpass.caching_strategy import fill_cache
+from .overpass.overpass import fill_cache
 from .cache import client as cache_client
 
 
diff --git a/backend/src/overpass/caching_strategy.py b/backend/src/overpass/caching_strategy.py
index d2c5add..2f6e944 100644
--- a/backend/src/overpass/caching_strategy.py
+++ b/backend/src/overpass/caching_strategy.py
@@ -3,7 +3,6 @@ import json
 import hashlib
 
 from ..constants import OSM_CACHE_DIR, OSM_TYPES
-from .overpass import Overpass
 
 
 def get_cache_key(query: str) -> str:
@@ -13,6 +12,7 @@ def get_cache_key(query: str) -> str:
     """
     return hashlib.md5(query.encode('utf-8')).hexdigest()
 
+
 class CachingStrategyBase:
     """
     Base class for implementing caching strategies.
@@ -25,15 +25,10 @@ class CachingStrategyBase:
         """Store data in the cache with the specified key."""
         raise NotImplementedError('Subclass should implement set')
 
-    def set_hollow(self, key, cell: tuple, osm_types: OSM_TYPES,
-                    selector: str, conditions=[], out='center'):
+    def set_hollow(self, key, **kwargs):
         """Create a hollow (empty) cache entry with a specific key."""
         raise NotImplementedError('Subclass should implement set_hollow')
 
-    def fill_hollow(self, key, value):
-        """Fill in the cache for an existing hollow entry."""
-        raise NotImplementedError('Subclass should implement fill_hollow')
-
     def close(self):
         """Clean up or close any resources used by the caching strategy."""
 
@@ -134,27 +129,3 @@ class CachingStrategy:
                     selector: str, conditions: list=None, out='center'):
         """Create a hollow cache entry."""
         cls.__strategy.set_hollow(key, cell, osm_types, selector, conditions, out)
-
-
-def fill_cache():
-    """
-    Scans the specified cache directory for files starting with 'hollow_' and attempts to load
-    their contents as JSON to fill the cache of the Overpass system.
-    """
-    overpass = Overpass()
-
-    with os.scandir(OSM_CACHE_DIR) as it:
-        for entry in it:
-            if entry.is_file() and entry.name.startswith('hollow_'):
-
-                try :
-                    # Read the whole file content as a string
-                    with open(entry.path, 'r') as f:
-                        # load data and fill the cache with the query and key
-                        json_data = json.load(f)
-                        overpass.fill_cache(json_data)
-                    # Now delete the file as the cache is filled
-                    os.remove(entry.path)
-
-                except Exception as exc :
-                    overpass.logger.error(f'An error occured while parsing file {f} as .json file')
diff --git a/backend/src/overpass/overpass.py b/backend/src/overpass/overpass.py
index 17a1031..f2158f6 100644
--- a/backend/src/overpass/overpass.py
+++ b/backend/src/overpass/overpass.py
@@ -52,10 +52,11 @@ class Overpass :
 
         # If there is no missing data, return the cached responses
         if not hollow_cache_keys :
-            self.logger.info('Cache hit.')
+            self.logger.info(f'Cache hit for {len(cached_responses)} quadrants.')
             return self._combine_cached_data(cached_responses)
 
         # TODO If there is SOME missing data : hybrid stuff with partial cache
+        self.logger.info(f'Cache miss for {len(hollow_cache_keys)} quadrants.')
 
         # Missing data: Make a query to Overpass API
         query_str = Overpass.build_query(bbox, osm_types, selector, conditions, out)
@@ -80,9 +81,7 @@ class Overpass :
                 response_data = response.read().decode('utf-8')  # Convert the HTTPResponse to a string
                 data = json.loads(response_data)  # Load the JSON from the string
                 elements = data.get('elements', [])
-
-                self.logger.info(f'Cache miss. Fetching data through Overpass.')
-                self.logger.debug(f'Query = {query_str}')
+                # self.logger.debug(f'Query = {query_str}')
                 return elements
 
         except urllib.error.URLError as e:
@@ -321,3 +320,27 @@ def get_base_info(elem: dict, osm_type: OSM_TYPES, with_name=False) :
         return osm_id, coords, name
     else :
         return osm_id, coords
+
+
+def fill_cache():
+    """
+    Scans the specified cache directory for files starting with 'hollow_' and attempts to load
+    their contents as JSON to fill the cache of the Overpass system.
+    """
+    overpass = Overpass()
+
+    with os.scandir(OSM_CACHE_DIR) as it:
+        for entry in it:
+            if entry.is_file() and entry.name.startswith('hollow_'):
+
+                try :
+                    # Read the whole file content as a string
+                    with open(entry.path, 'r') as f:
+                        # load data and fill the cache with the query and key
+                        json_data = json.load(f)
+                        overpass.fill_cache(json_data)
+                    # Now delete the file as the cache is filled
+                    os.remove(entry.path)
+
+                except Exception as exc :
+                    overpass.logger.error(f'An error occured while parsing file {entry.path} as .json file')
