amazing cache #55
										
											
												File diff suppressed because one or more lines are too long
											
										
									
								
							| @@ -29,8 +29,99 @@ class Overpass : | ||||
|         self.caching_strategy = CachingStrategy.use(caching_strategy, cache_dir=cache_dir) | ||||
|  | ||||
|  | ||||
|     @classmethod | ||||
|     def build_query(self, bbox: tuple, osm_types: OSM_TYPES, | ||||
|     def send_query(self, bbox: tuple, osm_types: OSM_TYPES, | ||||
|                     selector: str, conditions=[], out='center') -> ET: | ||||
|         """ | ||||
|         Sends the Overpass QL query to the Overpass API and returns the parsed JSON response. | ||||
|  | ||||
|         Args: | ||||
|             query (str): The Overpass QL query to be sent to the Overpass API. | ||||
|  | ||||
|         Returns: | ||||
|             dict: The parsed JSON response from the Overpass API, or None if the request fails. | ||||
|         """ | ||||
|         # Determine which grid cells overlap with this bounding box. | ||||
|         overlapping_cells = Overpass.get_overlapping_cells(bbox) | ||||
|  | ||||
|         # Check the cache for any data that overlaps with these cells | ||||
|         cell_key_dict = {} | ||||
|         for cell in overlapping_cells : | ||||
|             for elem in osm_types : | ||||
|                 key_str = f"{elem}[{selector}]{conditions}({','.join(map(str, cell))})" | ||||
|              | ||||
|             cell_key_dict[cell] = get_cache_key(key_str) | ||||
|  | ||||
|         cached_responses = [] | ||||
|         hollow_cache_keys = [] | ||||
|  | ||||
|         # Retrieve the cached data and mark the missing entries as hollow | ||||
|         for cell, key in cell_key_dict.items(): | ||||
|             cached_data = self.caching_strategy.get(key) | ||||
|             if cached_data is not None : | ||||
|                 cached_responses.append(cached_data) | ||||
|             else: | ||||
|                 # Cache miss: Mark the cache key as hollow | ||||
|                 self.caching_strategy.set_hollow(key, cell, osm_types, selector, conditions, out) | ||||
|                 hollow_cache_keys.append(key) | ||||
|  | ||||
|         # If there is no missing data, return the cached responses | ||||
|         if not hollow_cache_keys : | ||||
|             self.logger.debug(f'Cache hit.') | ||||
|             return self.combine_cached_data(cached_responses) | ||||
|          | ||||
|         # TODO If there is SOME missing data : hybrid stuff with partial cache | ||||
|          | ||||
|         # Build the query string in case of needed overpass query | ||||
|         query_str = Overpass.build_query(bbox, osm_types, selector, conditions, out) | ||||
|  | ||||
|         # Prepare the data to be sent as POST request, encoded as bytes | ||||
|         data = urllib.parse.urlencode({'data': query_str}).encode('utf-8') | ||||
|  | ||||
|         try: | ||||
|             # Create a Request object with the specified URL, data, and headers | ||||
|             request = urllib.request.Request(self.overpass_url, data=data, headers=self.headers) | ||||
|  | ||||
|             # Send the request and read the response | ||||
|             with urllib.request.urlopen(request) as response: | ||||
|                 # Read and decode the response | ||||
|                 response_data = response.read().decode('utf-8') | ||||
|                 root = ET.fromstring(response_data) | ||||
|  | ||||
|                 self.logger.debug(f'Cache miss. Fetching data through Overpass\nQuery = {query_str}') | ||||
|  | ||||
|                 return root | ||||
|  | ||||
|         except urllib.error.URLError as e: | ||||
|             raise ConnectionError(f"Error connecting to Overpass API: {e}") from e | ||||
|  | ||||
|  | ||||
|     def fill_cache(self, xml_string: str) : | ||||
|          | ||||
|         # Build the query using info from hollow cache entry | ||||
|         query_str, cache_key = Overpass.build_query_from_hollow(xml_string) | ||||
|  | ||||
|         # Prepare the data to be sent as POST request, encoded as bytes | ||||
|         data = urllib.parse.urlencode({'data': query_str}).encode('utf-8') | ||||
|  | ||||
|         try: | ||||
|             # Create a Request object with the specified URL, data, and headers | ||||
|             request = urllib.request.Request(self.overpass_url, data=data, headers=self.headers) | ||||
|  | ||||
|             # Send the request and read the response | ||||
|             with urllib.request.urlopen(request) as response: | ||||
|                 # Read and decode the response | ||||
|                 response_data = response.read().decode('utf-8') | ||||
|                 root = ET.fromstring(response_data) | ||||
|  | ||||
|                 self.caching_strategy.set(cache_key, root) | ||||
|                 self.logger.debug(f'Cache set') | ||||
|  | ||||
|         except urllib.error.URLError as e: | ||||
|             raise ConnectionError(f"Error connecting to Overpass API: {e}") from e | ||||
|  | ||||
|  | ||||
|     @staticmethod | ||||
|     def build_query(bbox: tuple, osm_types: OSM_TYPES, | ||||
|                     selector: str, conditions=[], out='center') -> str: | ||||
|         """ | ||||
|         Constructs a query string for the Overpass API to retrieve OpenStreetMap (OSM) data. | ||||
| @@ -78,73 +169,8 @@ class Overpass : | ||||
|         return query | ||||
|  | ||||
|  | ||||
|     def send_query(self, bbox: tuple, osm_types: OSM_TYPES, | ||||
|                     selector: str, conditions=[], out='center') -> ET: | ||||
|         """ | ||||
|         Sends the Overpass QL query to the Overpass API and returns the parsed JSON response. | ||||
|  | ||||
|         Args: | ||||
|             query (str): The Overpass QL query to be sent to the Overpass API. | ||||
|  | ||||
|         Returns: | ||||
|             dict: The parsed JSON response from the Overpass API, or None if the request fails. | ||||
|         """ | ||||
|         # Determine which grid cells overlap with this bounding box. | ||||
|         overlapping_cells = self.get_overlapping_cells(bbox) | ||||
|  | ||||
|         # Check the cache for any data that overlaps with these cells | ||||
|         cell_key_dict = {} | ||||
|         for cell in overlapping_cells : | ||||
|             for elem in osm_types : | ||||
|                 key_str = f"{elem}[{selector}]{conditions}({','.join(map(str, cell))})" | ||||
|              | ||||
|             cell_key_dict[cell] = get_cache_key(key_str) | ||||
|  | ||||
|         cached_responses = [] | ||||
|         hollow_cache_keys = [] | ||||
|  | ||||
|         # Retrieve the cached data and mark the missing entries as hollow | ||||
|         for cell, key in cell_key_dict.items(): | ||||
|             cached_data = self.caching_strategy.get(key) | ||||
|             if cached_data is not None : | ||||
|                 cached_responses.append(cached_data) | ||||
|             else: | ||||
|                 # Cache miss: Mark the cache key as hollow | ||||
|                 self.caching_strategy.set_hollow(key, cell, osm_types, selector, conditions, out) | ||||
|                 hollow_cache_keys.append(key) | ||||
|  | ||||
|         # If there is no missing data, return the cached responses | ||||
|         if not hollow_cache_keys : | ||||
|             self.logger.debug(f'Cache hit.') | ||||
|             return self.combine_cached_data(cached_responses) | ||||
|          | ||||
|         # TODO If there is SOME missing data : hybrid stuff with partial cache | ||||
|          | ||||
|         # Build the query string in case of needed overpass query | ||||
|         query_str = self.build_query(bbox, osm_types, selector, conditions, out) | ||||
|  | ||||
|         # Prepare the data to be sent as POST request, encoded as bytes | ||||
|         data = urllib.parse.urlencode({'data': query_str}).encode('utf-8') | ||||
|  | ||||
|         try: | ||||
|             # Create a Request object with the specified URL, data, and headers | ||||
|             request = urllib.request.Request(self.overpass_url, data=data, headers=self.headers) | ||||
|  | ||||
|             # Send the request and read the response | ||||
|             with urllib.request.urlopen(request) as response: | ||||
|                 # Read and decode the response | ||||
|                 response_data = response.read().decode('utf-8') | ||||
|                 root = ET.fromstring(response_data) | ||||
|  | ||||
|                 self.logger.debug(f'Cache miss. Fetching data through Overpass\nQuery = {query_str}') | ||||
|  | ||||
|                 return root | ||||
|  | ||||
|         except urllib.error.URLError as e: | ||||
|             raise ConnectionError(f"Error connecting to Overpass API: {e}") from e | ||||
|  | ||||
|  | ||||
|     def build_query_from_hollow(self, xml_string): | ||||
|     @staticmethod | ||||
|     def build_query_from_hollow(xml_string): | ||||
|         """Extract variables from an XML string.""" | ||||
|          | ||||
|         # Parse the XML string into an ElementTree object | ||||
| @@ -153,17 +179,19 @@ class Overpass : | ||||
|         # Extract values from the XML tree | ||||
|         key = root.find('key').text | ||||
|         cell = tuple(map(float, root.find('cell').text.strip('()').split(','))) | ||||
|         bbox = self.get_bbox_from_grid_cell(cell[0], cell[1]) | ||||
|         bbox = Overpass.get_bbox_from_grid_cell(cell[0], cell[1]) | ||||
|         osm_types = root.find('osm_types').text.split(',') | ||||
|         selector = root.find('selector').text | ||||
|         conditions = root.find('conditions').text.split(',') if root.find('conditions').text != "none" else [] | ||||
|         out = root.find('out').text | ||||
|  | ||||
|         query_str = self.build_query(bbox, osm_types, selector, conditions, out) | ||||
|         query_str = Overpass.build_query(bbox, osm_types, selector, conditions, out) | ||||
|          | ||||
|         return query_str, key | ||||
|  | ||||
|     def get_grid_cell(self, lat: float, lon: float): | ||||
|  | ||||
|     @staticmethod | ||||
|     def get_grid_cell(lat: float, lon: float): | ||||
|         """ | ||||
|         Returns the grid cell coordinates for a given latitude and longitude. | ||||
|         Each grid cell is 0.05°lat x 0.05°lon resolution in size. | ||||
| @@ -173,7 +201,8 @@ class Overpass : | ||||
|         return (lat_index, lon_index) | ||||
|  | ||||
|  | ||||
|     def get_bbox_from_grid_cell(self, lat_index: int, lon_index: int): | ||||
|     @staticmethod | ||||
|     def get_bbox_from_grid_cell(lat_index: int, lon_index: int): | ||||
|         """ | ||||
|         Returns the bounding box for a given grid cell index. | ||||
|         Each grid cell is resolution x resolution in size. | ||||
| @@ -191,16 +220,16 @@ class Overpass : | ||||
|         return (min_lat, min_lon, max_lat, max_lon) | ||||
|  | ||||
|  | ||||
|  | ||||
|     def get_overlapping_cells(self, query_bbox: tuple): | ||||
|     @staticmethod | ||||
|     def get_overlapping_cells(query_bbox: tuple): | ||||
|         """ | ||||
|         Returns a set of all grid cells that overlap with the given bounding box. | ||||
|         """ | ||||
|         # Extract location from the query bbox | ||||
|         lat_min, lon_min, lat_max, lon_max = query_bbox | ||||
|  | ||||
|         min_lat_cell, min_lon_cell = self.get_grid_cell(lat_min, lon_min) | ||||
|         max_lat_cell, max_lon_cell = self.get_grid_cell(lat_max, lon_max) | ||||
|         min_lat_cell, min_lon_cell = Overpass.get_grid_cell(lat_min, lon_min) | ||||
|         max_lat_cell, max_lon_cell = Overpass.get_grid_cell(lat_max, lon_max) | ||||
|  | ||||
|         overlapping_cells = set() | ||||
|         for lat_idx in range(min_lat_cell, max_lat_cell + 1): | ||||
| @@ -210,7 +239,8 @@ class Overpass : | ||||
|         return overlapping_cells | ||||
|  | ||||
|  | ||||
|     def combine_cached_data(self, cached_data_list): | ||||
|     @staticmethod | ||||
|     def combine_cached_data(cached_data_list): | ||||
|         """ | ||||
|         Combines data from multiple cached responses into a single result. | ||||
|         """ | ||||
| @@ -221,30 +251,6 @@ class Overpass : | ||||
|         return combined_data | ||||
|      | ||||
|  | ||||
|     def fill_cache(self, query_str: str, cache_key) : | ||||
|          | ||||
|         # Prepare the data to be sent as POST request, encoded as bytes | ||||
|         data = urllib.parse.urlencode({'data': query_str}).encode('utf-8') | ||||
|  | ||||
|         try: | ||||
|             # Create a Request object with the specified URL, data, and headers | ||||
|             request = urllib.request.Request(self.overpass_url, data=data, headers=self.headers) | ||||
|  | ||||
|             # Send the request and read the response | ||||
|             with urllib.request.urlopen(request) as response: | ||||
|                 # Read and decode the response | ||||
|                 response_data = response.read().decode('utf-8') | ||||
|                 root = ET.fromstring(response_data) | ||||
|  | ||||
|                 self.caching_strategy.set(cache_key, root) | ||||
|                 self.logger.debug(f'Cache set') | ||||
|  | ||||
|         except urllib.error.URLError as e: | ||||
|             raise ConnectionError(f"Error connecting to Overpass API: {e}") from e | ||||
|  | ||||
|  | ||||
|  | ||||
|  | ||||
| def get_base_info(elem: ET.Element, osm_type: OSM_TYPES, with_name=False) : | ||||
|     """ | ||||
|     Extracts base information (coordinates, OSM ID, and optionally a name) from an OSM element. | ||||
| @@ -301,11 +307,8 @@ def fill_cache(): | ||||
|                 with open(entry.path, 'r') as f: | ||||
|                     xml_string = f.read()   | ||||
|  | ||||
|                 # Build the query and cache key from the hollow XML string | ||||
|                 query_str, key = overpass.build_query_from_hollow(xml_string) | ||||
|                  | ||||
|                 # Fill the cache with the query and key | ||||
|                 overpass.fill_cache(query_str, key) | ||||
|                 overpass.fill_cache(xml_string) | ||||
|  | ||||
|                 # Now delete the file as the cache is filled | ||||
|                 os.remove(entry.path) | ||||
|   | ||||
		Reference in New Issue
	
	Block a user