diff --git a/.github/workflows/microk8s-ci.yaml b/.github/workflows/microk8s-ci.yaml index dbd41e38f9..a6801fb7dc 100644 --- a/.github/workflows/microk8s-ci.yaml +++ b/.github/workflows/microk8s-ci.yaml @@ -17,6 +17,12 @@ jobs: btrix-microk8s-test: runs-on: ubuntu-latest steps: + - name: Initial Disk Cleanup + uses: mathio/gha-cleanup@v1 + with: + remove-browsers: true + verbose: true + - uses: balchua/microk8s-actions@v0.3.1 with: channel: "1.25/stable" diff --git a/backend/btrixcloud/main.py b/backend/btrixcloud/main.py index 0f32934dac..4fc8e5510e 100644 --- a/backend/btrixcloud/main.py +++ b/backend/btrixcloud/main.py @@ -181,7 +181,6 @@ def main() -> None: crawl_manager, invites, current_active_user, - shared_secret_or_superuser, ) init_subs_api(app, mdb, org_ops, user_manager, shared_secret_or_superuser) diff --git a/backend/btrixcloud/models.py b/backend/btrixcloud/models.py index b301be4818..77b6842d57 100644 --- a/backend/btrixcloud/models.py +++ b/backend/btrixcloud/models.py @@ -1829,6 +1829,8 @@ class S3Storage(BaseModel): REASON_PAUSED = "subscriptionPaused" REASON_CANCELED = "subscriptionCanceled" +SubscriptionEventType = Literal["create", "import", "update", "cancel", "add-minutes"] + # ============================================================================ class OrgQuotas(BaseModel): @@ -1857,8 +1859,6 @@ class OrgQuotasIn(BaseModel): extraExecMinutes: Optional[int] = None giftedExecMinutes: Optional[int] = None - context: str | None = None - # ============================================================================ class Plan(BaseModel): @@ -1883,6 +1883,7 @@ class SubscriptionEventOut(BaseModel): oid: UUID timestamp: datetime + type: SubscriptionEventType # ============================================================================ @@ -1948,18 +1949,55 @@ class SubscriptionCancel(BaseModel): # ============================================================================ -class SubscriptionTrialEndReminder(BaseModel): - """Email reminder that subscription will end soon""" +class SubscriptionCancelOut(SubscriptionCancel, SubscriptionEventOut): + """Output model for subscription cancellation event""" - subId: str - behavior_on_trial_end: Literal["cancel", "continue", "read-only"] + type: Literal["cancel"] = "cancel" # ============================================================================ -class SubscriptionCancelOut(SubscriptionCancel, SubscriptionEventOut): - """Output model for subscription cancellation event""" +class SubscriptionAddMinutes(BaseModel): + """Represents a purchase of additional minutes""" + + oid: UUID + minutes: int + total_price: float + currency: str + + context: str - type: Literal["cancel"] = "cancel" + +# ============================================================================ +class SubscriptionAddMinutesOut(SubscriptionAddMinutes, SubscriptionEventOut): + """SubscriptionAddMinutes output model""" + + type: Literal["add-minutes"] = "add-minutes" + + +# ============================================================================ +SubscriptionEventAny = Union[ + SubscriptionCreate, + SubscriptionUpdate, + SubscriptionCancel, + SubscriptionImport, + SubscriptionAddMinutes, +] + +SubscriptionEventAnyOut = Union[ + SubscriptionCreateOut, + SubscriptionUpdateOut, + SubscriptionCancelOut, + SubscriptionImportOut, + SubscriptionAddMinutesOut, +] + + +# ============================================================================ +class SubscriptionTrialEndReminder(BaseModel): + """Email reminder that subscription will end soon""" + + subId: str + behavior_on_trial_end: Literal["cancel", "continue", "read-only"] # ============================================================================ @@ -3126,14 +3164,7 @@ class PaginatedProfileResponse(PaginatedResponse): class PaginatedSubscriptionEventResponse(PaginatedResponse): """Response model for paginated subscription events""" - items: List[ - Union[ - SubscriptionCreateOut, - SubscriptionUpdateOut, - SubscriptionCancelOut, - SubscriptionImportOut, - ] - ] + items: List[SubscriptionEventAnyOut] # ============================================================================ diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py index 11f3b3f906..0db16323cc 100644 --- a/backend/btrixcloud/orgs.py +++ b/backend/btrixcloud/orgs.py @@ -625,8 +625,6 @@ async def update_quotas( ) -> None: """update organization quotas""" - quotas.context = None - previous_extra_mins = ( org.quotas.extraExecMinutes if (org.quotas and org.quotas.extraExecMinutes) @@ -1546,7 +1544,6 @@ def init_orgs_api( crawl_manager: CrawlManager, invites: InviteOps, user_dep: Callable[[str], Awaitable[User]], - superuser_or_shared_secret_dep: Callable[[str], Awaitable[User]], ): """Init organizations api router for /orgs""" # pylint: disable=too-many-locals,invalid-name @@ -1701,20 +1698,7 @@ async def update_quotas( if not user.is_superuser: raise HTTPException(status_code=403, detail="Not Allowed") - await ops.update_quotas(org, quotas, mode="set", context=quotas.context) - - return {"updated": True} - - @app.post( - "/orgs/{oid}/quotas/add", tags=["organizations"], response_model=UpdatedResponse - ) - async def update_quotas_add( - oid: UUID, - quotas: OrgQuotasIn, - _user: User = Depends(superuser_or_shared_secret_dep), - ): - org = await ops.get_org_by_id(oid) - await ops.update_quotas(org, quotas, mode="add", context=quotas.context) + await ops.update_quotas(org, quotas, mode="set") return {"updated": True} diff --git a/backend/btrixcloud/subs.py b/backend/btrixcloud/subs.py index a4e9c22157..ce86f022f5 100644 --- a/backend/btrixcloud/subs.py +++ b/backend/btrixcloud/subs.py @@ -2,13 +2,13 @@ Subscription API handling """ -from typing import Awaitable, Callable, Union, Any, Optional, Tuple, List +from typing import Awaitable, Callable, Any, Optional, Tuple, List, Annotated import os import asyncio from uuid import UUID from datetime import datetime -from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi import APIRouter, Depends, HTTPException, Request, Query import aiohttp from motor.motor_asyncio import AsyncIOMotorDatabase @@ -23,16 +23,22 @@ SubscriptionImport, SubscriptionUpdate, SubscriptionCancel, + SubscriptionAddMinutes, + SubscriptionEventAny, SubscriptionCreateOut, SubscriptionImportOut, SubscriptionUpdateOut, SubscriptionCancelOut, + SubscriptionAddMinutesOut, + SubscriptionEventAnyOut, + SubscriptionEventType, Subscription, SubscriptionPortalUrlRequest, SubscriptionPortalUrlResponse, SubscriptionCanceledResponse, SubscriptionTrialEndReminder, Organization, + OrgQuotasIn, InviteToOrgRequest, InviteAddedResponse, User, @@ -228,15 +234,22 @@ async def send_trial_end_reminder( return SuccessResponse(success=True) + async def add_sub_minutes(self, add_min: SubscriptionAddMinutes): + """add extra minutes for subscription""" + org = await self.org_ops.get_org_by_id(add_min.oid) + quotas = OrgQuotasIn(extraExecMinutes=add_min.minutes) + await self.org_ops.update_quotas( + org, quotas, mode="add", context=add_min.context + ) + + await self.add_sub_event("add-minutes", add_min, add_min.oid) + + return {"updated": True} + async def add_sub_event( self, - type_: str, - event: Union[ - SubscriptionCreate, - SubscriptionImport, - SubscriptionUpdate, - SubscriptionCancel, - ], + type_: SubscriptionEventType, + event: SubscriptionEventAny, oid: UUID, ) -> None: """add a subscription event to the db""" @@ -246,12 +259,9 @@ async def add_sub_event( data["oid"] = oid await self.subs.insert_one(data) - def _get_sub_by_type_from_data(self, data: dict[str, object]) -> Union[ - SubscriptionCreateOut, - SubscriptionImportOut, - SubscriptionUpdateOut, - SubscriptionCancelOut, - ]: + def _get_sub_by_type_from_data( + self, data: dict[str, object] + ) -> SubscriptionEventAnyOut: """convert dict to propert background job type""" if data["type"] == "create": return SubscriptionCreateOut(**data) @@ -259,7 +269,12 @@ def _get_sub_by_type_from_data(self, data: dict[str, object]) -> Union[ return SubscriptionImportOut(**data) if data["type"] == "update": return SubscriptionUpdateOut(**data) - return SubscriptionCancelOut(**data) + if data["type"] == "cancel": + return SubscriptionCancelOut(**data) + if data["type"] == "add-minutes": + return SubscriptionAddMinutesOut(**data) + + raise HTTPException(status_code=500, detail="unknown sub event") # pylint: disable=too-many-arguments async def list_sub_events( @@ -268,19 +283,13 @@ async def list_sub_events( sub_id: Optional[str] = None, oid: Optional[UUID] = None, plan_id: Optional[str] = None, + type_: Optional[SubscriptionEventType] = None, page_size: int = DEFAULT_PAGE_SIZE, page: int = 1, sort_by: Optional[str] = None, sort_direction: Optional[int] = -1, ) -> Tuple[ - List[ - Union[ - SubscriptionCreateOut, - SubscriptionImportOut, - SubscriptionUpdateOut, - SubscriptionCancelOut, - ] - ], + List[SubscriptionEventAnyOut], int, ]: """list subscription events""" @@ -298,6 +307,8 @@ async def list_sub_events( query["planId"] = plan_id if oid: query["oid"] = oid + if type_: + query["type"] = type_ aggregate = [{"$match": query}] @@ -515,6 +526,15 @@ async def send_trial_end_reminder( ): return await ops.send_trial_end_reminder(reminder) + @app.post( + "/subscriptions/add-minutes", + tags=["subscriptions"], + dependencies=[Depends(superuser_or_shared_secret_dep)], + response_model=UpdatedResponse, + ) + async def add_sub_minutes(add_min: SubscriptionAddMinutes): + return await ops.add_sub_minutes(add_min) + assert org_ops.router @app.get( @@ -540,6 +560,7 @@ async def get_sub_events( subId: Optional[str] = None, oid: Optional[UUID] = None, planId: Optional[str] = None, + type_: Annotated[Optional[SubscriptionEventType], Query(alias="type")] = None, pageSize: int = DEFAULT_PAGE_SIZE, page: int = 1, sortBy: Optional[str] = "timestamp", @@ -551,6 +572,7 @@ async def get_sub_events( oid=oid, plan_id=planId, page_size=pageSize, + type_=type_, page=page, sort_by=sortBy, sort_direction=sortDirection, diff --git a/backend/test/test_org_subs.py b/backend/test/test_org_subs.py index e17c2c35ef..a74a2cb64e 100644 --- a/backend/test/test_org_subs.py +++ b/backend/test/test_org_subs.py @@ -508,7 +508,6 @@ def test_subscription_events_log(admin_auth_headers, non_default_org_id): "planId": "basic2", "futureCancelDate": None, "quotas": { - "context": None, "maxPagesPerCrawl": 50, "storageQuota": 500000, "extraExecMinutes": None, @@ -577,7 +576,6 @@ def test_subscription_events_log_filter_sub_id(admin_auth_headers): "planId": "basic2", "futureCancelDate": None, "quotas": { - "context": None, "maxPagesPerCrawl": 50, "storageQuota": 500000, "extraExecMinutes": None, @@ -639,7 +637,6 @@ def test_subscription_events_log_filter_oid(admin_auth_headers): "planId": "basic2", "futureCancelDate": None, "quotas": { - "context": None, "maxPagesPerCrawl": 50, "storageQuota": 500000, "extraExecMinutes": None, @@ -675,7 +672,6 @@ def test_subscription_events_log_filter_plan_id(admin_auth_headers): "planId": "basic2", "futureCancelDate": None, "quotas": { - "context": None, "maxPagesPerCrawl": 50, "storageQuota": 500000, "extraExecMinutes": None, @@ -727,7 +723,6 @@ def test_subscription_events_log_filter_status(admin_auth_headers): "planId": "basic2", "futureCancelDate": None, "quotas": { - "context": None, "maxPagesPerCrawl": 50, "storageQuota": 500000, "extraExecMinutes": None, @@ -767,7 +762,7 @@ def test_subscription_events_log_filter_sort(admin_auth_headers): last_id = None for event in events: - sub_id = event["subId"] + sub_id = event.get("subId") if last_id: assert last_id <= sub_id last_id = sub_id @@ -783,7 +778,7 @@ def test_subscription_events_log_filter_sort(admin_auth_headers): last_id = None for event in events: - sub_id = event["subId"] + sub_id = event.get("subId") if last_id: assert last_id >= sub_id last_id = sub_id @@ -920,3 +915,57 @@ def test_subscription_events_log_filter_sort(admin_auth_headers): assert last_id >= cancel_date if cancel_date: last_date = cancel_date + + +def test_subscription_add_minutes(admin_auth_headers): + r = requests.post( + f"{API_PREFIX}/subscriptions/add-minutes", + headers=admin_auth_headers, + json={ + "oid": str(new_subs_oid_2), + "minutes": 75, + "total_price": 350, + "currency": "usd", + "context": "addon", + }, + ) + + assert r.status_code == 200 + assert r.json() == {"updated": True} + + # get event from log + r = requests.get( + f"{API_PREFIX}/subscriptions/events?oid={new_subs_oid_2}&type=add-minutes", + headers=admin_auth_headers, + ) + assert r.status_code == 200 + data = r.json() + assert len(data["items"]) == 1 + event = data["items"][0] + + assert event["type"] == "add-minutes" + assert event["oid"] == new_subs_oid_2 + assert event["minutes"] == 75 + assert event["total_price"] == 350 + assert event["currency"] == "usd" + assert event["context"] == "addon" + + # check org quota updates for corresponding entry + r = requests.get( + f"{API_PREFIX}/orgs/{new_subs_oid_2}", + headers=admin_auth_headers, + ) + + assert r.status_code == 200 + quota_updates = r.json()["quotaUpdates"] + assert len(quota_updates) + last_update = quota_updates[-1] + assert last_update["context"] == "addon" + assert last_update["update"] == { + "maxPagesPerCrawl": 100, + "storageQuota": 1000000, + "extraExecMinutes": 75, # only this value updated from previous + "giftedExecMinutes": 0, + "maxConcurrentCrawls": 1, + "maxExecMinutesPerMonth": 1000, + } diff --git a/backend/test_nightly/test_execution_minutes_quota.py b/backend/test_nightly/test_execution_minutes_quota.py index 992d732bae..978fbc6b16 100644 --- a/backend/test_nightly/test_execution_minutes_quota.py +++ b/backend/test_nightly/test_execution_minutes_quota.py @@ -196,36 +196,3 @@ def test_unset_execution_mins_quota(org_with_quotas, admin_auth_headers): ) data = r.json() assert data.get("updated") == True - - -def test_add_execution_mins_extra_quotas( - org_with_quotas, admin_auth_headers, preshared_secret_auth_headers -): - r = requests.post( - f"{API_PREFIX}/orgs/{org_with_quotas}/quotas/add", - headers=preshared_secret_auth_headers, - json={ - "extraExecMinutes": EXTRA_MINS_ADDED_QUOTA, - "context": "test context 123", - }, - ) - data = r.json() - assert data.get("updated") == True - - # Ensure org data looks as we expect - r = requests.get( - f"{API_PREFIX}/orgs/{org_with_quotas}", - headers=admin_auth_headers, - ) - data = r.json() - assert ( - data["extraExecSecondsAvailable"] == EXTRA_SECS_QUOTA + EXTRA_SECS_ADDED_QUOTA - ) - assert data["giftedExecSecondsAvailable"] == GIFTED_SECS_QUOTA - assert data["extraExecSeconds"] == {} - assert data["giftedExecSeconds"] == {} - assert len(data["quotaUpdates"]) - for update in data["quotaUpdates"]: - assert update["modified"] - assert update["update"] - assert data["quotaUpdates"][-1]["context"] == "test context 123" diff --git a/chart/test/microk8s-ci.yaml b/chart/test/microk8s-ci.yaml index 1d19cc6f4c..857faf44bf 100644 --- a/chart/test/microk8s-ci.yaml +++ b/chart/test/microk8s-ci.yaml @@ -14,3 +14,5 @@ frontend_pull_policy: "IfNotPresent" crawler_extra_cpu_per_browser: 300m crawler_extra_memory_per_browser: 256Mi + +backend_memory: "350Mi" diff --git a/chart/test/test.yaml b/chart/test/test.yaml index 1add7c53f8..292b1cf521 100644 --- a/chart/test/test.yaml +++ b/chart/test/test.yaml @@ -16,6 +16,9 @@ operator_resync_seconds: 3 qa_scale: 2 +backend_memory: "500Mi" + + # lower storage sizes redis_storage: "100Mi" profile_browser_workdir_size: "100Mi"