n0mad1k
103 lines
3.2 KiB
Python
103 lines
3.2 KiB
Python
"""Plain text / markdown parser for Mosaic."""
|
|
import re
|
|
import logging
|
|
|
|
from .base import ParseResult
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# ANSI escape pattern
|
|
ANSI_ESCAPE = re.compile(r'\x1b\[[0-9;]*[a-zA-Z]|\x1b\].*?\x07')
|
|
|
|
|
|
class TextDocParser:
|
|
"""Parse plain text and markdown documents."""
|
|
|
|
def parse(self, file_path: str) -> ParseResult:
|
|
"""Parse a plain text file."""
|
|
try:
|
|
# Try UTF-8 first, fall back to latin-1
|
|
try:
|
|
with open(file_path, 'r', encoding='utf-8') as f:
|
|
text = f.read()
|
|
except UnicodeDecodeError:
|
|
with open(file_path, 'r', encoding='latin-1') as f:
|
|
text = f.read()
|
|
|
|
# Strip ANSI escape sequences
|
|
text = ANSI_ESCAPE.sub('', text)
|
|
|
|
# Detect if it's markdown
|
|
is_markdown = self._detect_markdown(text, file_path)
|
|
|
|
# Extract basic metadata
|
|
metadata = {
|
|
'is_markdown': is_markdown,
|
|
}
|
|
|
|
# Try to extract a title
|
|
title = self._extract_title(text)
|
|
|
|
# Basic section detection
|
|
sections = self._detect_sections(text)
|
|
if sections:
|
|
metadata['section_count'] = len(sections)
|
|
|
|
return ParseResult(
|
|
success=True,
|
|
text=text.strip(),
|
|
metadata=metadata,
|
|
doc_type='text',
|
|
title=title,
|
|
char_count=len(text),
|
|
)
|
|
|
|
except Exception as e:
|
|
return ParseResult(error=f"Text parse error: {str(e)}")
|
|
|
|
def _detect_markdown(self, text: str, file_path: str) -> bool:
|
|
"""Detect if text is markdown."""
|
|
if file_path.endswith(('.md', '.markdown', '.mkd')):
|
|
return True
|
|
# Check for markdown indicators
|
|
md_patterns = [
|
|
r'^#{1,6}\s', # Headers
|
|
r'^\*\*[^*]+\*\*', # Bold
|
|
r'^\- ', # List items
|
|
r'^\d+\. ', # Numbered list
|
|
r'\[.+\]\(.+\)', # Links
|
|
]
|
|
matches = sum(1 for p in md_patterns if re.search(p, text[:2000], re.MULTILINE))
|
|
return matches >= 2
|
|
|
|
def _extract_title(self, text: str) -> str:
|
|
"""Extract title from first heading or first line."""
|
|
lines = text.strip().split('\n')
|
|
for line in lines[:5]:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
# Markdown heading
|
|
match = re.match(r'^#{1,3}\s+(.+)', line)
|
|
if match:
|
|
return match.group(1).strip()
|
|
# First non-empty line
|
|
if len(line) < 200:
|
|
return line
|
|
break
|
|
return ""
|
|
|
|
def _detect_sections(self, text: str) -> list[str]:
|
|
"""Detect section headings in text."""
|
|
sections = []
|
|
for line in text.split('\n'):
|
|
line = line.strip()
|
|
# Markdown headings
|
|
if re.match(r'^#{1,6}\s+', line):
|
|
sections.append(line)
|
|
# All-caps headings
|
|
elif (line.isupper() and 3 < len(line) < 100 and
|
|
not line.startswith(('---', '===', '***'))):
|
|
sections.append(line)
|
|
return sections
|