n0mad1k
126 lines
4.0 KiB
Python
126 lines
4.0 KiB
Python
"""HTML document parser for Mosaic — BS4 with html.parser backend."""
|
|
import re
|
|
import logging
|
|
|
|
from bs4 import BeautifulSoup
|
|
|
|
from .base import ParseResult
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Elements to remove (navigation, scripts, styles)
|
|
REMOVE_TAGS = ['script', 'style', 'nav', 'footer', 'header', 'aside',
|
|
'noscript', 'iframe', 'svg']
|
|
|
|
# ANSI escape sequence pattern
|
|
ANSI_ESCAPE = re.compile(r'\x1b\[[0-9;]*[a-zA-Z]|\x1b\].*?\x07')
|
|
|
|
|
|
class HTMLDocParser:
|
|
"""Parse HTML documents into structured text."""
|
|
|
|
def parse(self, file_path: str) -> ParseResult:
|
|
"""Parse an HTML file.
|
|
|
|
Uses BeautifulSoup with 'html.parser' backend (explicitly specified,
|
|
not auto-detected) to extract structured text.
|
|
"""
|
|
try:
|
|
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
|
|
raw_html = f.read()
|
|
|
|
soup = BeautifulSoup(raw_html, 'html.parser')
|
|
|
|
# Extract title
|
|
title = ""
|
|
title_tag = soup.find('title')
|
|
if title_tag:
|
|
title = title_tag.get_text(strip=True)
|
|
|
|
# Extract metadata
|
|
metadata = self._extract_metadata(soup)
|
|
|
|
# Remove unwanted elements
|
|
for tag_name in REMOVE_TAGS:
|
|
for tag in soup.find_all(tag_name):
|
|
tag.decompose()
|
|
|
|
# Extract text preserving structure
|
|
text = self._extract_structured_text(soup)
|
|
|
|
# Strip ANSI escape sequences
|
|
text = ANSI_ESCAPE.sub('', text)
|
|
|
|
# Extract tables separately
|
|
tables = self._extract_tables(soup)
|
|
if tables:
|
|
metadata['tables'] = tables
|
|
|
|
char_count = len(text)
|
|
|
|
return ParseResult(
|
|
success=True,
|
|
text=text,
|
|
metadata=metadata,
|
|
doc_type='html',
|
|
title=title,
|
|
char_count=char_count,
|
|
)
|
|
|
|
except Exception as e:
|
|
return ParseResult(error=f"HTML parse error: {str(e)}")
|
|
|
|
def _extract_metadata(self, soup: BeautifulSoup) -> dict:
|
|
"""Extract metadata from HTML meta tags."""
|
|
metadata = {}
|
|
for meta in soup.find_all('meta'):
|
|
name = meta.get('name', meta.get('property', ''))
|
|
content = meta.get('content', '')
|
|
if name and content:
|
|
metadata[name] = content
|
|
return metadata
|
|
|
|
def _extract_structured_text(self, soup: BeautifulSoup) -> str:
|
|
"""Extract text preserving heading structure."""
|
|
parts = []
|
|
|
|
for element in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p',
|
|
'li', 'td', 'th', 'pre', 'blockquote', 'div']):
|
|
text = element.get_text(separator=' ', strip=True)
|
|
if not text:
|
|
continue
|
|
|
|
tag = element.name
|
|
if tag.startswith('h'):
|
|
level = int(tag[1])
|
|
prefix = '#' * level
|
|
parts.append(f"\n{prefix} {text}\n")
|
|
elif tag == 'li':
|
|
parts.append(f" - {text}")
|
|
elif tag == 'pre':
|
|
parts.append(f"\n```\n{text}\n```\n")
|
|
elif tag == 'blockquote':
|
|
parts.append(f"> {text}")
|
|
else:
|
|
parts.append(text)
|
|
|
|
result = '\n'.join(parts)
|
|
# Collapse excessive whitespace
|
|
result = re.sub(r'\n{3,}', '\n\n', result)
|
|
return result.strip()
|
|
|
|
def _extract_tables(self, soup: BeautifulSoup) -> list[dict]:
|
|
"""Extract tables as structured data."""
|
|
tables = []
|
|
for table in soup.find_all('table'):
|
|
rows = []
|
|
for tr in table.find_all('tr'):
|
|
cells = []
|
|
for td in tr.find_all(['td', 'th']):
|
|
cells.append(td.get_text(strip=True))
|
|
if cells:
|
|
rows.append(cells)
|
|
if rows:
|
|
tables.append({'rows': rows})
|
|
return tables
|