cache later

backend/src/overpass/overpass.py

@@ -32,92 +32,74 @@ class Overpass :
     def send_query(self, bbox: tuple, osm_types: OSM_TYPES,
                     selector: str, conditions=[], out='center') -> ET:
         """
-        Sends the Overpass QL query to the Overpass API and returns the parsed JSON response.
+        Sends the Overpass QL query to the Overpass API and returns the parsed XML response.
 
         Args:
-            query (str): The Overpass QL query to be sent to the Overpass API.
+            bbox (tuple): Bounding box for the query.
+            osm_types (list[str]): List of OSM element types (e.g., 'node', 'way').
+            selector (str): Key or tag to filter OSM elements (e.g., 'highway').
+            conditions (list): Optional list of additional filter conditions in Overpass QL format.
+            out (str): Output format ('center', 'body', etc.). Defaults to 'center'.
 
         Returns:
-            dict: The parsed JSON response from the Overpass API, or None if the request fails.
+            ET.Element: Parsed XML response from the Overpass API, or cached data if available.
         """
         # Determine which grid cells overlap with this bounding box.
-        overlapping_cells = Overpass.get_overlapping_cells(bbox)
+        overlapping_cells = Overpass._get_overlapping_cells(bbox)
 
-        # Check the cache for any data that overlaps with these cells
-        cell_key_dict = {}
-        for cell in overlapping_cells :
-            for elem in osm_types :
-                key_str = f"{elem}[{selector}]{conditions}({','.join(map(str, cell))})"
-            
-            cell_key_dict[cell] = get_cache_key(key_str)
-
-        cached_responses = []
-        hollow_cache_keys = []
-
-        # Retrieve the cached data and mark the missing entries as hollow
-        for cell, key in cell_key_dict.items():
-            cached_data = self.caching_strategy.get(key)
-            if cached_data is not None :
-                cached_responses.append(cached_data)
-            else:
-                # Cache miss: Mark the cache key as hollow
-                self.caching_strategy.set_hollow(key, cell, osm_types, selector, conditions, out)
-                hollow_cache_keys.append(key)
+        # Retrieve cached data and identify missing cache entries
+        cached_responses, hollow_cache_keys = self._retrieve_cached_data(overlapping_cells, osm_types, selector, conditions, out)
 
         # If there is no missing data, return the cached responses
         if not hollow_cache_keys :
             self.logger.debug(f'Cache hit.')
-            return self.combine_cached_data(cached_responses)
+            return self._combine_cached_data(cached_responses)
         
         # TODO If there is SOME missing data : hybrid stuff with partial cache
         
-        # Build the query string in case of needed overpass query
+        # Missing data: Make a query to Overpass API
         query_str = Overpass.build_query(bbox, osm_types, selector, conditions, out)
+        self.fetch_data_from_api(query_str)
 
-        # Prepare the data to be sent as POST request, encoded as bytes
-        data = urllib.parse.urlencode({'data': query_str}).encode('utf-8')
 
+    def fetch_data_from_api(self, query_str: str, cache_key: str = None) -> ET.Element:
+        """
+        Fetch data from the Overpass API and update the cache.
+
+        Args:
+            query_str (str): The Overpass query string.
+            cached_responses (list): Cached responses to combine with fetched data.
+            hollow_cache_keys (list): Cache keys for missing data to be updated.
+
+        Returns:
+            ET.Element: Combined cached and fetched data.
+        """
         try:
-            # Create a Request object with the specified URL, data, and headers
+            data = urllib.parse.urlencode({'data': query_str}).encode('utf-8')
             request = urllib.request.Request(self.overpass_url, data=data, headers=self.headers)
 
-            # Send the request and read the response
             with urllib.request.urlopen(request) as response:
-                # Read and decode the response
                 response_data = response.read().decode('utf-8')
                 root = ET.fromstring(response_data)
 
-                self.logger.debug(f'Cache miss. Fetching data through Overpass\nQuery = {query_str}')
-
-                return root
+                if cache_key is not None :
+                    self.caching_strategy.set(cache_key, root)
+                    self.logger.debug(f'Cache set.')
+                else : 
+                    self.logger.debug(f'Cache miss. Fetching data through Overpass\nQuery = {query_str}')
+                    return root
 
         except urllib.error.URLError as e:
-            raise ConnectionError(f"Error connecting to Overpass API: {e}") from e
+            self.logger.error(f"Error connecting to Overpass API: {e}")
+            raise ConnectionError(f"Error connecting to Overpass API: {e}")
 
 
     def fill_cache(self, xml_string: str) :
-        
-        # Build the query using info from hollow cache entry
-        query_str, cache_key = Overpass.build_query_from_hollow(xml_string)
-
-        # Prepare the data to be sent as POST request, encoded as bytes
-        data = urllib.parse.urlencode({'data': query_str}).encode('utf-8')
-
-        try:
-            # Create a Request object with the specified URL, data, and headers
-            request = urllib.request.Request(self.overpass_url, data=data, headers=self.headers)
-
-            # Send the request and read the response
-            with urllib.request.urlopen(request) as response:
-                # Read and decode the response
-                response_data = response.read().decode('utf-8')
-                root = ET.fromstring(response_data)
-
-                self.caching_strategy.set(cache_key, root)
-                self.logger.debug(f'Cache set')
-
-        except urllib.error.URLError as e:
-            raise ConnectionError(f"Error connecting to Overpass API: {e}") from e
+        """
+        Fill cache with data by using a hollow cache entry's information.
+        """
+        query_str, cache_key = Overpass._build_query_from_hollow(xml_string)
+        self.fetch_data_from_api(query_str, cache_key)
 
 
     @staticmethod
@@ -169,17 +151,56 @@ class Overpass :
         return query
 
 
+    def _retrieve_cached_data(self, overlapping_cells: list, osm_types: OSM_TYPES, selector: str, conditions: list, out: str):
+        """
+        Retrieve cached data and identify missing cache entries.
+
+        Args:
+            overlapping_cells (list): Cells to check for cached data.
+            osm_types (list): OSM types (e.g., 'node', 'way').
+            selector (str): Key or tag to filter OSM elements.
+            conditions (list): Additional conditions to apply.
+            out (str): Output format.
+
+        Returns:
+            tuple: A tuple containing:
+                - cached_responses (list): List of cached data found.
+                - hollow_cache_keys (list): List of keys with missing data.
+        """
+        cell_key_dict = {}
+        for cell in overlapping_cells :
+            for elem in osm_types :
+                key_str = f"{elem}[{selector}]{conditions}({','.join(map(str, cell))})"
+            
+            cell_key_dict[cell] = get_cache_key(key_str)
+
+        cached_responses = []
+        hollow_cache_keys = []
+
+        # Retrieve the cached data and mark the missing entries as hollow
+        for cell, key in cell_key_dict.items():
+            cached_data = self.caching_strategy.get(key)
+            if cached_data is not None :
+                cached_responses.append(cached_data)
+            else:
+                self.caching_strategy.set_hollow(key, cell, osm_types, selector, conditions, out)
+                hollow_cache_keys.append(key)
+
+        return cached_responses, hollow_cache_keys
+
+
     @staticmethod
-    def build_query_from_hollow(xml_string):
-        """Extract variables from an XML string."""
-        
+    def _build_query_from_hollow(xml_string):
+        """
+        Build query string using information from a hollow cache entry.
+        """
         # Parse the XML string into an ElementTree object
         root = ET.fromstring(xml_string)
         
         # Extract values from the XML tree
         key = root.find('key').text
         cell = tuple(map(float, root.find('cell').text.strip('()').split(',')))
-        bbox = Overpass.get_bbox_from_grid_cell(cell[0], cell[1])
+        bbox = Overpass._get_bbox_from_grid_cell(cell[0], cell[1])
         osm_types = root.find('osm_types').text.split(',')
         selector = root.find('selector').text
         conditions = root.find('conditions').text.split(',') if root.find('conditions').text != "none" else []
@@ -191,7 +212,26 @@ class Overpass :
 
 
     @staticmethod
-    def get_grid_cell(lat: float, lon: float):
+    def _get_overlapping_cells(query_bbox: tuple):
+        """
+        Returns a set of all grid cells that overlap with the given bounding box.
+        """
+        # Extract location from the query bbox
+        lat_min, lon_min, lat_max, lon_max = query_bbox
+
+        min_lat_cell, min_lon_cell = Overpass._get_grid_cell(lat_min, lon_min)
+        max_lat_cell, max_lon_cell = Overpass._get_grid_cell(lat_max, lon_max)
+
+        overlapping_cells = set()
+        for lat_idx in range(min_lat_cell, max_lat_cell + 1):
+            for lon_idx in range(min_lon_cell, max_lon_cell + 1):
+                overlapping_cells.add((lat_idx, lon_idx))
+        
+        return overlapping_cells
+
+
+    @staticmethod
+    def _get_grid_cell(lat: float, lon: float):
         """
         Returns the grid cell coordinates for a given latitude and longitude.
         Each grid cell is 0.05°lat x 0.05°lon resolution in size.
@@ -202,7 +242,7 @@ class Overpass :
 
 
     @staticmethod
-    def get_bbox_from_grid_cell(lat_index: int, lon_index: int):
+    def _get_bbox_from_grid_cell(lat_index: int, lon_index: int):
         """
         Returns the bounding box for a given grid cell index.
         Each grid cell is resolution x resolution in size.
@@ -221,26 +261,7 @@ class Overpass :
 
 
     @staticmethod
-    def get_overlapping_cells(query_bbox: tuple):
-        """
-        Returns a set of all grid cells that overlap with the given bounding box.
-        """
-        # Extract location from the query bbox
-        lat_min, lon_min, lat_max, lon_max = query_bbox
-
-        min_lat_cell, min_lon_cell = Overpass.get_grid_cell(lat_min, lon_min)
-        max_lat_cell, max_lon_cell = Overpass.get_grid_cell(lat_max, lon_max)
-
-        overlapping_cells = set()
-        for lat_idx in range(min_lat_cell, max_lat_cell + 1):
-            for lon_idx in range(min_lon_cell, max_lon_cell + 1):
-                overlapping_cells.add((lat_idx, lon_idx))
-        
-        return overlapping_cells
-
-
-    @staticmethod
-    def combine_cached_data(cached_data_list):
+    def _combine_cached_data(cached_data_list):
         """
         Combines data from multiple cached responses into a single result.
         """