Merge pull request 'fix: saas restart logic' (#13) from furqan into dev
Reviewed-on: https://gitea.maskantech.in/gitea_admin/saas_backend/pulls/13
This commit is contained in:
+32
-9
@@ -31,21 +31,41 @@ logging.basicConfig(
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
async def wait_for_db(retries: int = 10, base_delay: float = 3.0, max_delay: float = 30.0):
|
||||||
@asynccontextmanager
|
"""Wait for the database to become available with exponential backoff."""
|
||||||
async def lifespan(app: FastAPI):
|
for attempt in range(1, retries + 1):
|
||||||
logger.info("Testing database connection...")
|
|
||||||
try:
|
try:
|
||||||
with engine.connect() as conn:
|
with engine.connect() as conn:
|
||||||
conn.execute(text("SELECT 1"))
|
conn.execute(text("SELECT 1"))
|
||||||
conn.commit()
|
|
||||||
logger.info("Database connection successful!")
|
logger.info("Database connection successful!")
|
||||||
|
return
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Database connection failed: {e}")
|
if attempt == retries:
|
||||||
|
logger.error(f"Database unreachable after {retries} attempts: {e}")
|
||||||
raise
|
raise
|
||||||
|
delay = min(base_delay * (2 ** (attempt - 1)), max_delay)
|
||||||
|
logger.warning(
|
||||||
|
f"Database not ready (attempt {attempt}/{retries}): {e}. "
|
||||||
|
f"Retrying in {delay:.0f}s..."
|
||||||
|
)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def lifespan(app: FastAPI):
|
||||||
|
logger.info("Starting SaaS application...")
|
||||||
|
|
||||||
|
await wait_for_db()
|
||||||
|
|
||||||
|
app.state.redis_available = False
|
||||||
|
|
||||||
|
try:
|
||||||
await redis_client.connect()
|
await redis_client.connect()
|
||||||
await run_in_threadpool(sync_redis_client.connect)
|
await run_in_threadpool(sync_redis_client.connect)
|
||||||
|
app.state.redis_available = True
|
||||||
|
logger.info("Redis connected successfully.")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Redis unavailable at startup: {e}")
|
||||||
|
logger.warning("Continuing without Redis — event queue disabled.")
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
f"{settings.PROJECT_NAME} v{settings.VERSION} started ({settings.APP_ENV})"
|
f"{settings.PROJECT_NAME} v{settings.VERSION} started ({settings.APP_ENV})"
|
||||||
@@ -55,7 +75,7 @@ async def lifespan(app: FastAPI):
|
|||||||
logger.info("Redis Event Consumer STARTED")
|
logger.info("Redis Event Consumer STARTED")
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
if not redis_client.client:
|
if not app.state.redis_available or not redis_client.client:
|
||||||
await asyncio.sleep(5)
|
await asyncio.sleep(5)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -69,7 +89,7 @@ async def lifespan(app: FastAPI):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error processing event {event_id}: {e}")
|
logger.error(f"Error processing event {event_id}: {e}")
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
await asyncio.sleep(1)
|
await asyncio.sleep(1)
|
||||||
|
|
||||||
async def fallback_poller():
|
async def fallback_poller():
|
||||||
@@ -94,9 +114,12 @@ async def lifespan(app: FastAPI):
|
|||||||
for task in background_tasks:
|
for task in background_tasks:
|
||||||
task.cancel()
|
task.cancel()
|
||||||
await asyncio.gather(*background_tasks, return_exceptions=True)
|
await asyncio.gather(*background_tasks, return_exceptions=True)
|
||||||
|
if app.state.redis_available:
|
||||||
|
try:
|
||||||
await redis_client.close()
|
await redis_client.close()
|
||||||
await run_in_threadpool(sync_redis_client.close)
|
await run_in_threadpool(sync_redis_client.close)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
def create_app() -> FastAPI:
|
def create_app() -> FastAPI:
|
||||||
app = FastAPI(
|
app = FastAPI(
|
||||||
|
|||||||
Reference in New Issue
Block a user