Initial commit: Phase 1 & Phase 2 infrastructure complete
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
"""
|
||||
GigLez Core Package
|
||||
|
||||
Shared core functionality for both deployment modes
|
||||
"""
|
||||
@@ -0,0 +1,17 @@
|
||||
"""
|
||||
GigLez Storage Package
|
||||
|
||||
Storage abstraction supporting filesystem and S3-compatible object storage
|
||||
"""
|
||||
|
||||
from .base import StorageBackend
|
||||
from .filesystem import LocalStorage
|
||||
from .s3 import S3Storage
|
||||
from .factory import get_storage_backend
|
||||
|
||||
__all__ = [
|
||||
'StorageBackend',
|
||||
'LocalStorage',
|
||||
'S3Storage',
|
||||
'get_storage_backend'
|
||||
]
|
||||
@@ -0,0 +1,124 @@
|
||||
"""
|
||||
Storage Backend Base Class
|
||||
|
||||
Abstract interface for storage implementations
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Optional, BinaryIO
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class StorageBackend(ABC):
|
||||
"""
|
||||
Abstract base class for storage backends
|
||||
|
||||
Implementations:
|
||||
- LocalStorage: Filesystem storage (development)
|
||||
- S3Storage: S3-compatible object storage (production)
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def save_file(self, file_hash: str, content: bytes, filename: Optional[str] = None) -> str:
|
||||
"""
|
||||
Save file to storage
|
||||
|
||||
Args:
|
||||
file_hash: SHA256 hash of file (used as key)
|
||||
content: File content as bytes
|
||||
filename: Original filename (optional, for metadata)
|
||||
|
||||
Returns:
|
||||
Storage path/URL where file was saved
|
||||
|
||||
Raises:
|
||||
StorageError: If save fails
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_file(self, file_hash: str) -> bytes:
|
||||
"""
|
||||
Retrieve file from storage
|
||||
|
||||
Args:
|
||||
file_hash: SHA256 hash of file
|
||||
|
||||
Returns:
|
||||
File content as bytes
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If file doesn't exist
|
||||
StorageError: If retrieval fails
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def file_exists(self, file_hash: str) -> bool:
|
||||
"""
|
||||
Check if file exists in storage
|
||||
|
||||
Args:
|
||||
file_hash: SHA256 hash of file
|
||||
|
||||
Returns:
|
||||
True if file exists, False otherwise
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def delete_file(self, file_hash: str) -> bool:
|
||||
"""
|
||||
Delete file from storage
|
||||
|
||||
Args:
|
||||
file_hash: SHA256 hash of file
|
||||
|
||||
Returns:
|
||||
True if deleted, False if file didn't exist
|
||||
|
||||
Raises:
|
||||
StorageError: If deletion fails
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_file_url(self, file_hash: str) -> str:
|
||||
"""
|
||||
Get URL for accessing file
|
||||
|
||||
Args:
|
||||
file_hash: SHA256 hash of file
|
||||
|
||||
Returns:
|
||||
URL string (file:// for local, https:// for S3)
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_file_path(self, file_hash: str) -> str:
|
||||
"""
|
||||
Get storage path for file hash
|
||||
|
||||
Uses content-addressed storage pattern: first 2+2 chars for sharding
|
||||
Example: abc123...def -> ab/c1/abc123...def.sub
|
||||
|
||||
Args:
|
||||
file_hash: SHA256 hash (64 hex characters)
|
||||
|
||||
Returns:
|
||||
Relative storage path
|
||||
"""
|
||||
if len(file_hash) < 4:
|
||||
raise ValueError(f"Invalid file hash: {file_hash}")
|
||||
|
||||
# Shard by first 2+2 characters (prevents too many files in one directory)
|
||||
shard1 = file_hash[:2]
|
||||
shard2 = file_hash[2:4]
|
||||
filename = f"{file_hash}.sub"
|
||||
|
||||
return f"{shard1}/{shard2}/{filename}"
|
||||
|
||||
|
||||
class StorageError(Exception):
|
||||
"""Base exception for storage errors"""
|
||||
pass
|
||||
@@ -0,0 +1,64 @@
|
||||
"""
|
||||
Storage Backend Factory
|
||||
|
||||
Creates appropriate storage backend based on configuration
|
||||
"""
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from config.settings import settings
|
||||
from .base import StorageBackend
|
||||
from .filesystem import LocalStorage
|
||||
from .s3 import S3Storage
|
||||
|
||||
|
||||
def get_storage_backend() -> StorageBackend:
|
||||
"""
|
||||
Get storage backend based on configuration
|
||||
|
||||
Returns:
|
||||
StorageBackend instance (LocalStorage or S3Storage)
|
||||
|
||||
Raises:
|
||||
ValueError: If storage type is invalid
|
||||
"""
|
||||
|
||||
if settings.use_local_storage:
|
||||
logger.info(f"Using LocalStorage: {settings.storage_path}")
|
||||
return LocalStorage(base_path=settings.storage_path)
|
||||
|
||||
elif settings.use_s3_storage:
|
||||
logger.info(f"Using S3Storage: bucket={settings.storage_bucket}")
|
||||
|
||||
if not settings.aws_access_key_id or not settings.aws_secret_access_key:
|
||||
raise ValueError("AWS credentials not configured for S3 storage")
|
||||
|
||||
return S3Storage(
|
||||
bucket=settings.storage_bucket,
|
||||
access_key=settings.aws_access_key_id,
|
||||
secret_key=settings.aws_secret_access_key,
|
||||
endpoint=settings.s3_endpoint,
|
||||
region=settings.aws_region
|
||||
)
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown storage type: {settings.storage_type}")
|
||||
|
||||
|
||||
# Create a singleton instance for convenience
|
||||
_storage_instance = None
|
||||
|
||||
|
||||
def get_storage() -> StorageBackend:
|
||||
"""
|
||||
Get singleton storage instance
|
||||
|
||||
Returns:
|
||||
Global storage backend instance
|
||||
"""
|
||||
global _storage_instance
|
||||
|
||||
if _storage_instance is None:
|
||||
_storage_instance = get_storage_backend()
|
||||
|
||||
return _storage_instance
|
||||
@@ -0,0 +1,153 @@
|
||||
"""
|
||||
Local Filesystem Storage Backend
|
||||
|
||||
For development mode (Termux) - stores files in local directory
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from loguru import logger
|
||||
|
||||
from .base import StorageBackend, StorageError
|
||||
|
||||
|
||||
class LocalStorage(StorageBackend):
|
||||
"""
|
||||
Local filesystem storage implementation
|
||||
|
||||
Stores files in organized directory structure:
|
||||
storage/
|
||||
├── ab/
|
||||
│ ├── c1/
|
||||
│ │ └── abc123...def.sub
|
||||
"""
|
||||
|
||||
def __init__(self, base_path: str = "./storage"):
|
||||
"""
|
||||
Initialize local storage
|
||||
|
||||
Args:
|
||||
base_path: Base directory for file storage
|
||||
"""
|
||||
self.base_path = Path(base_path)
|
||||
|
||||
# Create base directory if it doesn't exist
|
||||
self.base_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info(f"LocalStorage initialized at: {self.base_path.absolute()}")
|
||||
|
||||
def save_file(self, file_hash: str, content: bytes, filename: Optional[str] = None) -> str:
|
||||
"""Save file to local filesystem"""
|
||||
try:
|
||||
# Get storage path
|
||||
relative_path = self.get_file_path(file_hash)
|
||||
full_path = self.base_path / relative_path
|
||||
|
||||
# Create parent directories
|
||||
full_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Write file
|
||||
with open(full_path, 'wb') as f:
|
||||
f.write(content)
|
||||
|
||||
logger.debug(f"File saved: {relative_path} ({len(content)} bytes)")
|
||||
|
||||
return str(relative_path)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save file {file_hash}: {e}")
|
||||
raise StorageError(f"Failed to save file: {e}")
|
||||
|
||||
def get_file(self, file_hash: str) -> bytes:
|
||||
"""Retrieve file from local filesystem"""
|
||||
try:
|
||||
relative_path = self.get_file_path(file_hash)
|
||||
full_path = self.base_path / relative_path
|
||||
|
||||
if not full_path.exists():
|
||||
raise FileNotFoundError(f"File not found: {file_hash}")
|
||||
|
||||
with open(full_path, 'rb') as f:
|
||||
content = f.read()
|
||||
|
||||
logger.debug(f"File retrieved: {relative_path} ({len(content)} bytes)")
|
||||
|
||||
return content
|
||||
|
||||
except FileNotFoundError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to retrieve file {file_hash}: {e}")
|
||||
raise StorageError(f"Failed to retrieve file: {e}")
|
||||
|
||||
def file_exists(self, file_hash: str) -> bool:
|
||||
"""Check if file exists"""
|
||||
try:
|
||||
relative_path = self.get_file_path(file_hash)
|
||||
full_path = self.base_path / relative_path
|
||||
return full_path.exists()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to check file existence {file_hash}: {e}")
|
||||
return False
|
||||
|
||||
def delete_file(self, file_hash: str) -> bool:
|
||||
"""Delete file from filesystem"""
|
||||
try:
|
||||
relative_path = self.get_file_path(file_hash)
|
||||
full_path = self.base_path / relative_path
|
||||
|
||||
if not full_path.exists():
|
||||
return False
|
||||
|
||||
full_path.unlink()
|
||||
logger.debug(f"File deleted: {relative_path}")
|
||||
|
||||
# Clean up empty parent directories
|
||||
try:
|
||||
full_path.parent.rmdir() # Only works if empty
|
||||
full_path.parent.parent.rmdir() # Only works if empty
|
||||
except OSError:
|
||||
pass # Directory not empty, that's fine
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete file {file_hash}: {e}")
|
||||
raise StorageError(f"Failed to delete file: {e}")
|
||||
|
||||
def get_file_url(self, file_hash: str) -> str:
|
||||
"""Get file:// URL for local file"""
|
||||
relative_path = self.get_file_path(file_hash)
|
||||
full_path = self.base_path / relative_path
|
||||
|
||||
return f"file://{full_path.absolute()}"
|
||||
|
||||
def get_storage_stats(self) -> dict:
|
||||
"""
|
||||
Get storage statistics
|
||||
|
||||
Returns:
|
||||
Dictionary with storage stats
|
||||
"""
|
||||
total_files = 0
|
||||
total_size = 0
|
||||
|
||||
try:
|
||||
for file_path in self.base_path.rglob("*.sub"):
|
||||
total_files += 1
|
||||
total_size += file_path.stat().st_size
|
||||
|
||||
return {
|
||||
"total_files": total_files,
|
||||
"total_size_bytes": total_size,
|
||||
"total_size_mb": round(total_size / (1024 * 1024), 2),
|
||||
"base_path": str(self.base_path.absolute())
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get storage stats: {e}")
|
||||
return {
|
||||
"error": str(e)
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
"""
|
||||
S3-Compatible Object Storage Backend
|
||||
|
||||
For production mode - stores files in S3 or MinIO
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
from loguru import logger
|
||||
|
||||
from .base import StorageBackend, StorageError
|
||||
|
||||
|
||||
class S3Storage(StorageBackend):
|
||||
"""
|
||||
S3-compatible object storage implementation
|
||||
|
||||
Supports:
|
||||
- AWS S3
|
||||
- MinIO
|
||||
- Any S3-compatible storage
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
bucket: str,
|
||||
access_key: str,
|
||||
secret_key: str,
|
||||
endpoint: Optional[str] = None,
|
||||
region: str = "us-east-1"
|
||||
):
|
||||
"""
|
||||
Initialize S3 storage
|
||||
|
||||
Args:
|
||||
bucket: S3 bucket name
|
||||
access_key: AWS access key ID
|
||||
secret_key: AWS secret access key
|
||||
endpoint: Custom endpoint (for MinIO, etc.)
|
||||
region: AWS region
|
||||
"""
|
||||
self.bucket = bucket
|
||||
self.access_key = access_key
|
||||
self.secret_key = secret_key
|
||||
self.endpoint = endpoint
|
||||
self.region = region
|
||||
|
||||
# Import boto3 lazily (only in production)
|
||||
try:
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
self.boto3 = boto3
|
||||
self.ClientError = ClientError
|
||||
|
||||
# Create S3 client
|
||||
self.s3_client = boto3.client(
|
||||
's3',
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
endpoint_url=endpoint,
|
||||
region_name=region
|
||||
)
|
||||
|
||||
# Verify bucket exists
|
||||
try:
|
||||
self.s3_client.head_bucket(Bucket=bucket)
|
||||
logger.info(f"S3Storage initialized: bucket={bucket}, region={region}")
|
||||
except self.ClientError:
|
||||
logger.error(f"S3 bucket not accessible: {bucket}")
|
||||
raise StorageError(f"S3 bucket not accessible: {bucket}")
|
||||
|
||||
except ImportError:
|
||||
raise StorageError("boto3 not installed. Install with: pip install boto3")
|
||||
|
||||
def save_file(self, file_hash: str, content: bytes, filename: Optional[str] = None) -> str:
|
||||
"""Upload file to S3"""
|
||||
try:
|
||||
# Get storage key
|
||||
key = self.get_file_path(file_hash)
|
||||
|
||||
# Upload to S3
|
||||
self.s3_client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=content,
|
||||
ContentType='application/octet-stream',
|
||||
Metadata={
|
||||
'original_filename': filename or '',
|
||||
'file_hash': file_hash
|
||||
}
|
||||
)
|
||||
|
||||
logger.debug(f"File uploaded to S3: {key} ({len(content)} bytes)")
|
||||
|
||||
return key
|
||||
|
||||
except self.ClientError as e:
|
||||
logger.error(f"Failed to upload file to S3 {file_hash}: {e}")
|
||||
raise StorageError(f"Failed to upload file: {e}")
|
||||
|
||||
def get_file(self, file_hash: str) -> bytes:
|
||||
"""Download file from S3"""
|
||||
try:
|
||||
key = self.get_file_path(file_hash)
|
||||
|
||||
# Download from S3
|
||||
response = self.s3_client.get_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key
|
||||
)
|
||||
|
||||
content = response['Body'].read()
|
||||
|
||||
logger.debug(f"File downloaded from S3: {key} ({len(content)} bytes)")
|
||||
|
||||
return content
|
||||
|
||||
except self.ClientError as e:
|
||||
if e.response['Error']['Code'] == 'NoSuchKey':
|
||||
raise FileNotFoundError(f"File not found: {file_hash}")
|
||||
logger.error(f"Failed to download file from S3 {file_hash}: {e}")
|
||||
raise StorageError(f"Failed to download file: {e}")
|
||||
|
||||
def file_exists(self, file_hash: str) -> bool:
|
||||
"""Check if file exists in S3"""
|
||||
try:
|
||||
key = self.get_file_path(file_hash)
|
||||
|
||||
self.s3_client.head_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
except self.ClientError as e:
|
||||
if e.response['Error']['Code'] == '404':
|
||||
return False
|
||||
logger.error(f"Failed to check file existence in S3 {file_hash}: {e}")
|
||||
return False
|
||||
|
||||
def delete_file(self, file_hash: str) -> bool:
|
||||
"""Delete file from S3"""
|
||||
try:
|
||||
key = self.get_file_path(file_hash)
|
||||
|
||||
# Check if file exists first
|
||||
if not self.file_exists(file_hash):
|
||||
return False
|
||||
|
||||
# Delete from S3
|
||||
self.s3_client.delete_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key
|
||||
)
|
||||
|
||||
logger.debug(f"File deleted from S3: {key}")
|
||||
|
||||
return True
|
||||
|
||||
except self.ClientError as e:
|
||||
logger.error(f"Failed to delete file from S3 {file_hash}: {e}")
|
||||
raise StorageError(f"Failed to delete file: {e}")
|
||||
|
||||
def get_file_url(self, file_hash: str, expires_in: int = 3600) -> str:
|
||||
"""
|
||||
Get presigned URL for file access
|
||||
|
||||
Args:
|
||||
file_hash: SHA256 hash of file
|
||||
expires_in: URL expiration time in seconds (default: 1 hour)
|
||||
|
||||
Returns:
|
||||
Presigned URL for file access
|
||||
"""
|
||||
try:
|
||||
key = self.get_file_path(file_hash)
|
||||
|
||||
# Generate presigned URL
|
||||
url = self.s3_client.generate_presigned_url(
|
||||
'get_object',
|
||||
Params={
|
||||
'Bucket': self.bucket,
|
||||
'Key': key
|
||||
},
|
||||
ExpiresIn=expires_in
|
||||
)
|
||||
|
||||
return url
|
||||
|
||||
except self.ClientError as e:
|
||||
logger.error(f"Failed to generate presigned URL for {file_hash}: {e}")
|
||||
raise StorageError(f"Failed to generate URL: {e}")
|
||||
|
||||
def get_storage_stats(self) -> dict:
|
||||
"""
|
||||
Get storage statistics from S3
|
||||
|
||||
Returns:
|
||||
Dictionary with storage stats
|
||||
"""
|
||||
try:
|
||||
# List all objects with .sub extension
|
||||
paginator = self.s3_client.get_paginator('list_objects_v2')
|
||||
pages = paginator.paginate(Bucket=self.bucket, Prefix='')
|
||||
|
||||
total_files = 0
|
||||
total_size = 0
|
||||
|
||||
for page in pages:
|
||||
if 'Contents' in page:
|
||||
for obj in page['Contents']:
|
||||
if obj['Key'].endswith('.sub'):
|
||||
total_files += 1
|
||||
total_size += obj['Size']
|
||||
|
||||
return {
|
||||
"total_files": total_files,
|
||||
"total_size_bytes": total_size,
|
||||
"total_size_mb": round(total_size / (1024 * 1024), 2),
|
||||
"bucket": self.bucket,
|
||||
"region": self.region
|
||||
}
|
||||
|
||||
except self.ClientError as e:
|
||||
logger.error(f"Failed to get S3 storage stats: {e}")
|
||||
return {
|
||||
"error": str(e)
|
||||
}
|
||||
Reference in New Issue
Block a user