better cache

backend/src/overpass/overpass.py

@@ -29,8 +29,99 @@ class Overpass :
         self.caching_strategy = CachingStrategy.use(caching_strategy, cache_dir=cache_dir)
 
 
-    @classmethod
+    def send_query(self, bbox: tuple, osm_types: OSM_TYPES,
-    def build_query(self, bbox: tuple, osm_types: OSM_TYPES,
+                    selector: str, conditions=[], out='center') -> ET:
+        """
+        Sends the Overpass QL query to the Overpass API and returns the parsed JSON response.
+
+        Args:
+            query (str): The Overpass QL query to be sent to the Overpass API.
+
+        Returns:
+            dict: The parsed JSON response from the Overpass API, or None if the request fails.
+        """
+        # Determine which grid cells overlap with this bounding box.
+        overlapping_cells = Overpass.get_overlapping_cells(bbox)
+
+        # Check the cache for any data that overlaps with these cells
+        cell_key_dict = {}
+        for cell in overlapping_cells :
+            for elem in osm_types :
+                key_str = f"{elem}[{selector}]{conditions}({','.join(map(str, cell))})"
+            
+            cell_key_dict[cell] = get_cache_key(key_str)
+
+        cached_responses = []
+        hollow_cache_keys = []
+
+        # Retrieve the cached data and mark the missing entries as hollow
+        for cell, key in cell_key_dict.items():
+            cached_data = self.caching_strategy.get(key)
+            if cached_data is not None :
+                cached_responses.append(cached_data)
+            else:
+                # Cache miss: Mark the cache key as hollow
+                self.caching_strategy.set_hollow(key, cell, osm_types, selector, conditions, out)
+                hollow_cache_keys.append(key)
+
+        # If there is no missing data, return the cached responses
+        if not hollow_cache_keys :
+            self.logger.debug(f'Cache hit.')
+            return self.combine_cached_data(cached_responses)
+        
+        # TODO If there is SOME missing data : hybrid stuff with partial cache
+        
+        # Build the query string in case of needed overpass query
+        query_str = Overpass.build_query(bbox, osm_types, selector, conditions, out)
+
+        # Prepare the data to be sent as POST request, encoded as bytes
+        data = urllib.parse.urlencode({'data': query_str}).encode('utf-8')
+
+        try:
+            # Create a Request object with the specified URL, data, and headers
+            request = urllib.request.Request(self.overpass_url, data=data, headers=self.headers)
+
+            # Send the request and read the response
+            with urllib.request.urlopen(request) as response:
+                # Read and decode the response
+                response_data = response.read().decode('utf-8')
+                root = ET.fromstring(response_data)
+
+                self.logger.debug(f'Cache miss. Fetching data through Overpass\nQuery = {query_str}')
+
+                return root
+
+        except urllib.error.URLError as e:
+            raise ConnectionError(f"Error connecting to Overpass API: {e}") from e
+
+
+    def fill_cache(self, xml_string: str) :
+        
+        # Build the query using info from hollow cache entry
+        query_str, cache_key = Overpass.build_query_from_hollow(xml_string)
+
+        # Prepare the data to be sent as POST request, encoded as bytes
+        data = urllib.parse.urlencode({'data': query_str}).encode('utf-8')
+
+        try:
+            # Create a Request object with the specified URL, data, and headers
+            request = urllib.request.Request(self.overpass_url, data=data, headers=self.headers)
+
+            # Send the request and read the response
+            with urllib.request.urlopen(request) as response:
+                # Read and decode the response
+                response_data = response.read().decode('utf-8')
+                root = ET.fromstring(response_data)
+
+                self.caching_strategy.set(cache_key, root)
+                self.logger.debug(f'Cache set')
+
+        except urllib.error.URLError as e:
+            raise ConnectionError(f"Error connecting to Overpass API: {e}") from e
+
+
+    @staticmethod
+    def build_query(bbox: tuple, osm_types: OSM_TYPES,
                     selector: str, conditions=[], out='center') -> str:
         """
         Constructs a query string for the Overpass API to retrieve OpenStreetMap (OSM) data.
@@ -78,73 +169,8 @@ class Overpass :
         return query
 
 
-    def send_query(self, bbox: tuple, osm_types: OSM_TYPES,
+    @staticmethod
-                    selector: str, conditions=[], out='center') -> ET:
+    def build_query_from_hollow(xml_string):
-        """
-        Sends the Overpass QL query to the Overpass API and returns the parsed JSON response.
-
-        Args:
-            query (str): The Overpass QL query to be sent to the Overpass API.
-
-        Returns:
-            dict: The parsed JSON response from the Overpass API, or None if the request fails.
-        """
-        # Determine which grid cells overlap with this bounding box.
-        overlapping_cells = self.get_overlapping_cells(bbox)
-
-        # Check the cache for any data that overlaps with these cells
-        cell_key_dict = {}
-        for cell in overlapping_cells :
-            for elem in osm_types :
-                key_str = f"{elem}[{selector}]{conditions}({','.join(map(str, cell))})"
-            
-            cell_key_dict[cell] = get_cache_key(key_str)
-
-        cached_responses = []
-        hollow_cache_keys = []
-
-        # Retrieve the cached data and mark the missing entries as hollow
-        for cell, key in cell_key_dict.items():
-            cached_data = self.caching_strategy.get(key)
-            if cached_data is not None :
-                cached_responses.append(cached_data)
-            else:
-                # Cache miss: Mark the cache key as hollow
-                self.caching_strategy.set_hollow(key, cell, osm_types, selector, conditions, out)
-                hollow_cache_keys.append(key)
-
-        # If there is no missing data, return the cached responses
-        if not hollow_cache_keys :
-            self.logger.debug(f'Cache hit.')
-            return self.combine_cached_data(cached_responses)
-        
-        # TODO If there is SOME missing data : hybrid stuff with partial cache
-        
-        # Build the query string in case of needed overpass query
-        query_str = self.build_query(bbox, osm_types, selector, conditions, out)
-
-        # Prepare the data to be sent as POST request, encoded as bytes
-        data = urllib.parse.urlencode({'data': query_str}).encode('utf-8')
-
-        try:
-            # Create a Request object with the specified URL, data, and headers
-            request = urllib.request.Request(self.overpass_url, data=data, headers=self.headers)
-
-            # Send the request and read the response
-            with urllib.request.urlopen(request) as response:
-                # Read and decode the response
-                response_data = response.read().decode('utf-8')
-                root = ET.fromstring(response_data)
-
-                self.logger.debug(f'Cache miss. Fetching data through Overpass\nQuery = {query_str}')
-
-                return root
-
-        except urllib.error.URLError as e:
-            raise ConnectionError(f"Error connecting to Overpass API: {e}") from e
-
-
-    def build_query_from_hollow(self, xml_string):
         """Extract variables from an XML string."""
         
         # Parse the XML string into an ElementTree object
@@ -153,17 +179,19 @@ class Overpass :
         # Extract values from the XML tree
         key = root.find('key').text
         cell = tuple(map(float, root.find('cell').text.strip('()').split(',')))
-        bbox = self.get_bbox_from_grid_cell(cell[0], cell[1])
+        bbox = Overpass.get_bbox_from_grid_cell(cell[0], cell[1])
         osm_types = root.find('osm_types').text.split(',')
         selector = root.find('selector').text
         conditions = root.find('conditions').text.split(',') if root.find('conditions').text != "none" else []
         out = root.find('out').text
 
-        query_str = self.build_query(bbox, osm_types, selector, conditions, out)
+        query_str = Overpass.build_query(bbox, osm_types, selector, conditions, out)
         
         return query_str, key
 
-    def get_grid_cell(self, lat: float, lon: float):
+
+    @staticmethod
+    def get_grid_cell(lat: float, lon: float):
         """
         Returns the grid cell coordinates for a given latitude and longitude.
         Each grid cell is 0.05°lat x 0.05°lon resolution in size.
@@ -173,7 +201,8 @@ class Overpass :
         return (lat_index, lon_index)
 
 
-    def get_bbox_from_grid_cell(self, lat_index: int, lon_index: int):
+    @staticmethod
+    def get_bbox_from_grid_cell(lat_index: int, lon_index: int):
         """
         Returns the bounding box for a given grid cell index.
         Each grid cell is resolution x resolution in size.
@@ -191,16 +220,16 @@ class Overpass :
         return (min_lat, min_lon, max_lat, max_lon)
 
 
-
+    @staticmethod
-    def get_overlapping_cells(self, query_bbox: tuple):
+    def get_overlapping_cells(query_bbox: tuple):
         """
         Returns a set of all grid cells that overlap with the given bounding box.
         """
         # Extract location from the query bbox
         lat_min, lon_min, lat_max, lon_max = query_bbox
 
-        min_lat_cell, min_lon_cell = self.get_grid_cell(lat_min, lon_min)
+        min_lat_cell, min_lon_cell = Overpass.get_grid_cell(lat_min, lon_min)
-        max_lat_cell, max_lon_cell = self.get_grid_cell(lat_max, lon_max)
+        max_lat_cell, max_lon_cell = Overpass.get_grid_cell(lat_max, lon_max)
 
         overlapping_cells = set()
         for lat_idx in range(min_lat_cell, max_lat_cell + 1):
@@ -210,7 +239,8 @@ class Overpass :
         return overlapping_cells
 
 
-    def combine_cached_data(self, cached_data_list):
+    @staticmethod
+    def combine_cached_data(cached_data_list):
         """
         Combines data from multiple cached responses into a single result.
         """
@@ -221,30 +251,6 @@ class Overpass :
         return combined_data
     
 
-    def fill_cache(self, query_str: str, cache_key) :
-        
-        # Prepare the data to be sent as POST request, encoded as bytes
-        data = urllib.parse.urlencode({'data': query_str}).encode('utf-8')
-
-        try:
-            # Create a Request object with the specified URL, data, and headers
-            request = urllib.request.Request(self.overpass_url, data=data, headers=self.headers)
-
-            # Send the request and read the response
-            with urllib.request.urlopen(request) as response:
-                # Read and decode the response
-                response_data = response.read().decode('utf-8')
-                root = ET.fromstring(response_data)
-
-                self.caching_strategy.set(cache_key, root)
-                self.logger.debug(f'Cache set')
-
-        except urllib.error.URLError as e:
-            raise ConnectionError(f"Error connecting to Overpass API: {e}") from e
-
-
-
-
 def get_base_info(elem: ET.Element, osm_type: OSM_TYPES, with_name=False) :
     """
     Extracts base information (coordinates, OSM ID, and optionally a name) from an OSM element.
@@ -301,11 +307,8 @@ def fill_cache():
                 with open(entry.path, 'r') as f:
                     xml_string = f.read()  
 
-                # Build the query and cache key from the hollow XML string
-                query_str, key = overpass.build_query_from_hollow(xml_string)
-                
                 # Fill the cache with the query and key
-                overpass.fill_cache(query_str, key)
+                overpass.fill_cache(xml_string)
 
                 # Now delete the file as the cache is filled
                 os.remove(entry.path)