anydev/structured-wikivoyage-exports
2
0
Fork 0
mirror of https://github.com/bcye/structured-wikivoyage-exports.git synced 2025-06-07 00:14:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix newlines being stripped from the resulting documents

Browse Source
This commit is contained in:
Bruce Röttgers 2025-04-30 13:32:52 +02:00
parent 804ffeb82b
commit 3330667fa7
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
transform-documents.py
View File

@ -129,16 +129,18 @@ class WikiDumpHandler(xml.sax.ContentHandler):
self.currentTag = None
def characters(self, content):
if not content.strip():
return
# Only filter whitespace for ID fields, preserve all content for text
if (
self.currentTag == "id"
and self.inPage
and not self.inRevision
and not self.currentPageId
):
self.currentPageId = content.strip()
content_stripped = content.strip()
if content_stripped: # Only process non-empty ID content
self.currentPageId = content_stripped
elif self.inText:
# Always append text content, even if it's just whitespace or newlines
self.currentText.append(content)
async def _process(self, text: str, uid: str):