Release v1.0.0
Some checks failed
CI / Checks (push) Failing after 13m2s

This commit is contained in:
2025-08-20 21:00:50 +02:00
commit b4338e2769
401 changed files with 23576 additions and 0 deletions

View File

@@ -0,0 +1,61 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
import uuid
from django.db import models
from hotpocket_backend.apps.saves.models import Save
from hotpocket_common.constants import AssociationsSearchMode
from hotpocket_soa.dto.associations import AssociationsQuery
class SaveAdapter:
def get_for_processing(self, *, pk: uuid.UUID) -> Save | None:
return Save.active_objects.get(pk=pk)
class AssociationAdapter:
def get_search_term_filters(self, *, query: AssociationsQuery) -> list[models.Q]:
# I suck at naming things, LOL.
result = [
(
models.Q(target__url__icontains=query.search)
|
models.Q(target_title__icontains=query.search)
|
models.Q(target__title__icontains=query.search)
|
models.Q(target_description__icontains=query.search)
|
models.Q(target__description__icontains=query.search)
),
]
return result
def get_search_filters(self, *, query: AssociationsQuery) -> list[models.Q]:
result = [
models.Q(account_uuid=query.account_uuid),
models.Q(target__deleted_at__isnull=True),
]
if query.mode == AssociationsSearchMode.ARCHIVED:
result.append(models.Q(archived_at__isnull=False))
else:
result.append(models.Q(archived_at__isnull=True))
match query.mode:
case AssociationsSearchMode.STARRED:
result.append(models.Q(starred_at__isnull=False))
if query.before is not None:
result.append(models.Q(pk__lt=query.before))
if query.after is not None:
result.append(models.Q(pk__gte=query.after))
if query.search is not None:
result.extend(self.get_search_term_filters(query=query))
return result

View File

@@ -0,0 +1,12 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from .base import AssociationAdapter, SaveAdapter
class BasicSaveAdapter(SaveAdapter):
pass
class BasicAssociationAdapter(AssociationAdapter):
pass

View File

@@ -0,0 +1,49 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
import logging
import uuid
import django.db
from django.db import models
from hotpocket_backend.apps.saves.models import Save
from hotpocket_common.db import postgres # noqa: F401
from hotpocket_soa.dto.associations import AssociationsQuery
from .base import AssociationAdapter, SaveAdapter
LOGGER = logging.getLogger(__name__)
class PostgresSaveAdapter(SaveAdapter):
ROW_LOCKED_MESSAGE = 'could not obtain lock on row in relation "saves_save"'
def get_for_processing(self, *, pk: uuid.UUID) -> Save | None:
try:
return Save.active_objects.select_for_update(nowait=True).get(pk=pk)
except django.db.utils.OperationalError as exception:
if exception.args[0].startswith(self.ROW_LOCKED_MESSAGE) is True:
LOGGER.info('Trying to process a locked save: pk=`%s`', pk)
return None
raise exception
class PostgresAssociationAdapter(AssociationAdapter):
def get_search_term_filters(self, *, query: AssociationsQuery) -> list[models.Q]:
result = [
(
models.Q(target__url__ilike=query.search)
|
models.Q(target_title__ilike=query.search)
|
models.Q(target__title__ilike=query.search)
|
models.Q(target_description__ilike=query.search)
|
models.Q(target__description__ilike=query.search)
),
]
return result

View File

@@ -0,0 +1 @@
from . import save # noqa: F401

View File

@@ -0,0 +1,29 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from django.contrib import admin
from django.utils.translation import gettext_lazy as _
from hotpocket_backend.apps.saves.models import Save
class SaveAdmin(admin.ModelAdmin):
list_display = (
'pk', 'key', 'account_uuid', 'created_at', 'render_is_active',
)
ordering = ['-created_at']
def has_delete_permission(self, request, obj=None):
return request.user.is_superuser
@admin.display(
description=_('Is Active?'), boolean=True, ordering='-deleted_at',
)
def render_is_active(self, obj: Save | None = None) -> bool | None:
if obj is None:
return None
return obj.is_active
admin.site.register(Save, SaveAdmin)

View File

@@ -0,0 +1,12 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from django.apps import AppConfig
from django.utils.translation import gettext_lazy as _
class CoreConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
label = 'saves'
name = 'hotpocket_backend.apps.saves'
verbose_name = _('Saves')

View File

@@ -0,0 +1,76 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from argparse import ArgumentParser
import logging
from django.core.management import BaseCommand
import django.db
from hotpocket_backend.apps.saves.models import Association
from hotpocket_backend.apps.saves.services import AssociationsService
from hotpocket_soa.dto.associations import AssociationUpdateIn
LOGGER = logging.getLogger(__name__)
class Command(BaseCommand):
help = 'Data migration for BTHLABS-38'
def add_arguments(self, parser: ArgumentParser):
parser.add_argument(
'-d', '--dry-run', action='store_true', default=False,
help='Dry run',
)
def handle(self, *args, **options):
dry_run = options.get('dry_run', False)
counters = {
'migrated': 0,
'errors': 0,
}
associations_service = AssociationsService()
with django.db.transaction.atomic():
for association in Association.objects.all():
LOGGER.info('Migrating: `%s`', association)
try:
update = AssociationUpdateIn.model_validate(dict(
target_title=(
association.target_title
if association.target_title is not None
else association.target_meta.get('title', None)
),
target_description=(
association.target_description
if association.target_description is not None
else association.target_meta.get('description', None)
),
))
_ = associations_service.update(
pk=association.pk,
update=update,
)
counters['migrated'] = counters['migrated'] + 1
except Exception as exception:
LOGGER.error(
'Unhandled exception: pk=`%s`: %s',
association.pk,
exception,
exc_info=exception,
)
counters['errors'] = counters['errors'] + 1
LOGGER.info(
'Done. Migrated: `%d`. Errors: `%d`',
counters['migrated'],
counters['errors'],
)
if dry_run is True:
raise RuntimeError('DRY RUN')

View File

@@ -0,0 +1,31 @@
# Generated by Django 5.2.3 on 2025-07-02 11:15
import uuid6
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Save',
fields=[
('id', models.UUIDField(default=uuid6.uuid7, editable=False, primary_key=True, serialize=False)),
('account_uuid', models.UUIDField(db_index=True, default=None)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('deleted_at', models.DateTimeField(blank=True, db_index=True, default=None, null=True)),
('key', models.CharField(db_index=True, default=None)),
('url', models.CharField(default=None)),
],
options={
'verbose_name': 'Save',
'verbose_name_plural': 'Saves',
},
),
]

View File

@@ -0,0 +1,36 @@
# Generated by Django 5.2.3 on 2025-07-03 08:33
import django.db.models.deletion
import uuid6
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('saves', '0001_initial'),
]
operations = [
migrations.AlterField(
model_name='save',
name='deleted_at',
field=models.DateTimeField(blank=True, db_index=True, default=None, editable=False, null=True),
),
migrations.CreateModel(
name='Association',
fields=[
('id', models.UUIDField(default=uuid6.uuid7, editable=False, primary_key=True, serialize=False)),
('account_uuid', models.UUIDField(db_index=True, default=None)),
('created_at', models.DateTimeField(auto_now_add=True)),
('updated_at', models.DateTimeField(auto_now=True)),
('deleted_at', models.DateTimeField(blank=True, db_index=True, default=None, editable=False, null=True)),
('archived_at', models.DateTimeField(blank=True, db_index=True, default=None, editable=False, null=True)),
('target', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='+', to='saves.save')),
],
options={
'verbose_name': 'Association',
'verbose_name_plural': 'Associations',
},
),
]

View File

@@ -0,0 +1,39 @@
# Generated by Django 5.2.3 on 2025-07-03 20:15
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('saves', '0002_alter_save_deleted_at_association'),
]
operations = [
migrations.AddField(
model_name='save',
name='content',
field=models.BinaryField(blank=True, default=None, null=True),
),
migrations.AddField(
model_name='save',
name='description',
field=models.CharField(blank=True, default=None, null=True),
),
migrations.AddField(
model_name='save',
name='last_processed_at',
field=models.DateTimeField(blank=True, default=None, null=True),
),
migrations.AddField(
model_name='save',
name='title',
field=models.CharField(blank=True, default=None, null=True),
),
migrations.AlterField(
model_name='association',
name='target',
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='associations', to='saves.save'),
),
]

View File

@@ -0,0 +1,18 @@
# Generated by Django 5.2.3 on 2025-07-14 06:01
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('saves', '0003_save_content_save_description_save_last_processed_at_and_more'),
]
operations = [
migrations.AddField(
model_name='association',
name='starred_at',
field=models.DateTimeField(blank=True, db_index=True, default=None, editable=False, null=True),
),
]

View File

@@ -0,0 +1,23 @@
# Generated by Django 5.2.3 on 2025-07-17 18:46
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('saves', '0004_association_starred_at'),
]
operations = [
migrations.AddField(
model_name='association',
name='target_meta',
field=models.JSONField(default=dict),
),
migrations.AddField(
model_name='save',
name='is_netloc_banned',
field=models.BooleanField(default=False),
),
]

View File

@@ -0,0 +1,18 @@
# Generated by Django 5.2.3 on 2025-07-19 09:37
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('saves', '0005_association_target_meta_save_is_netloc_banned'),
]
operations = [
migrations.AlterField(
model_name='association',
name='target_meta',
field=models.JSONField(blank=True, default=dict),
),
]

View File

@@ -0,0 +1,23 @@
# Generated by Django 5.2.3 on 2025-07-26 12:36
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('saves', '0006_alter_association_target_meta'),
]
operations = [
migrations.AddField(
model_name='association',
name='target_description',
field=models.CharField(blank=True, db_index=True, default=None, null=True),
),
migrations.AddField(
model_name='association',
name='target_title',
field=models.CharField(blank=True, db_index=True, default=None, null=True),
),
]

View File

@@ -0,0 +1,2 @@
from .association import Association # noqa: F401
from .save import Save # noqa: F401

View File

@@ -0,0 +1,72 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from django.db import models
from django.utils.translation import gettext_lazy as _
from hotpocket_backend.apps.core.models import Model
class ActiveAssociationsManager(models.Manager):
def get_queryset(self) -> models.QuerySet[Association]:
return super().get_queryset().filter(
deleted_at__isnull=True,
)
class Association(Model):
archived_at = models.DateTimeField(
auto_now=False,
auto_now_add=False,
blank=True,
null=True,
default=None,
db_index=True,
editable=False,
)
starred_at = models.DateTimeField(
auto_now=False,
auto_now_add=False,
blank=True,
null=True,
default=None,
db_index=True,
editable=False,
)
target_meta = models.JSONField(
blank=True,
null=False,
default=dict,
)
target_title = models.CharField(
blank=True,
null=True,
default=None,
db_index=True,
)
target_description = models.CharField(
blank=True,
null=True,
default=None,
db_index=True,
)
target = models.ForeignKey(
'saves.Save',
blank=False,
null=False,
default=None,
db_index=True,
related_name='associations',
on_delete=models.CASCADE,
)
objects = models.Manager()
active_objects = ActiveAssociationsManager()
class Meta:
verbose_name = _('Association')
verbose_name_plural = _('Associations')
def __str__(self) -> str:
return f'<Association pk={self.pk}>'

View File

@@ -0,0 +1,52 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
from django.db import models
from django.utils.translation import gettext_lazy as _
from hotpocket_backend.apps.core.models import Model
class ActiveSavesManager(models.Manager):
def get_queryset(self) -> models.QuerySet[Save]:
return super().get_queryset().filter(
deleted_at__isnull=True,
)
class Save(Model):
key = models.CharField(
blank=False, null=False, default=None, db_index=True,
)
url = models.CharField(
blank=False, null=False, default=None,
)
content = models.BinaryField(
blank=True, null=True, default=None, editable=False,
)
title = models.CharField(
blank=True, null=True, default=None,
)
description = models.CharField(
blank=True, null=True, default=None,
)
last_processed_at = models.DateTimeField(
auto_now=False,
auto_now_add=False,
blank=True,
null=True,
default=None,
)
is_netloc_banned = models.BooleanField(
blank=False, null=False, default=False,
)
objects = models.Manager()
active_objects = ActiveSavesManager()
class Meta:
verbose_name = _('Save')
verbose_name_plural = _('Saves')
def __str__(self) -> str:
return f'<Save pk={self.pk} key={self.key}>'

View File

@@ -0,0 +1,3 @@
from .associations import AssociationsService # noqa: F401
from .save_processor import SaveProcessorService # noqa: F401
from .saves import SavesService # noqa: F401

View File

@@ -0,0 +1,160 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
import datetime
import logging
import uuid
from django.db import models
from django.utils.timezone import now
from hotpocket_backend.apps.core.services import get_adapter
from hotpocket_backend.apps.saves.models import Association, Save
from hotpocket_backend.apps.saves.types import PAssociationAdapter
from hotpocket_soa.dto.associations import (
AssociationsQuery,
AssociationUpdateIn,
)
from .saves import SavesService
LOGGER = logging.getLogger(__name__)
class AssociationsService:
class AssociationsServiceError(Exception):
pass
class AssociationNotFound(AssociationsServiceError):
pass
@property
def adapter(self) -> PAssociationAdapter:
if hasattr(self, '_adapter') is False:
adapter_klass = get_adapter(
'SAVES_ASSOCIATION_ADAPTER',
'hotpocket_backend.apps.saves.adapters.basic:BasicAssociationAdapter',
)
self._adapter = adapter_klass()
return self._adapter
def create(self,
*,
account_uuid: uuid.UUID,
save_uuid: uuid.UUID,
pk: uuid.UUID | None = None,
created_at: datetime.datetime | None = None,
) -> Association:
save = SavesService().get(pk=save_uuid)
defaults = dict(
account_uuid=account_uuid,
target=save,
)
if pk is not None:
defaults['id'] = pk
result, created = Association.objects.get_or_create(
account_uuid=account_uuid,
deleted_at__isnull=True,
target=save,
archived_at__isnull=True,
defaults=defaults,
)
if created is True:
if created_at is not None:
result.created_at = created_at
result.save()
return result
def get(self,
*,
pk: uuid.UUID,
with_target: bool = False,
) -> Association:
try:
query_set = Association.active_objects.\
filter(
target__deleted_at__isnull=True,
)
if with_target is True:
query_set = query_set.select_related('target')
return query_set.get(pk=pk)
except Association.DoesNotExist as exception:
raise self.AssociationNotFound(
f'Association not found: pk=`{pk}`',
) from exception
def search(self,
*,
query: AssociationsQuery,
offset: int = 0,
limit: int = 10,
order_by: str = '-pk',
) -> models.QuerySet[Save]:
filters = self.adapter.get_search_filters(query=query)
result = Association.active_objects.\
select_related('target').\
filter(*filters).\
order_by(order_by)
return result[offset:offset + limit]
def update(self,
*,
pk: uuid.UUID,
update: AssociationUpdateIn,
) -> Association:
association = self.get(pk=pk)
association.target_title = update.target_title
association.target_description = update.target_description
next_target_meta = {
**(association.target_meta or {}),
}
next_target_meta.pop('title', None)
next_target_meta.pop('description', None)
association.target_meta = next_target_meta
association.save()
return association
def archive(self, *, pk: uuid.UUID) -> bool:
association = self.get(pk=pk)
association.archived_at = now()
association.save()
return True
def star(self, *, pk: uuid.UUID) -> Association:
association = self.get(pk=pk)
if association.starred_at is None:
association.starred_at = now()
association.save()
return association
def unstar(self, *, pk: uuid.UUID) -> Association:
association = self.get(pk=pk)
if association.starred_at is not None:
association.starred_at = None
association.save()
return association
def delete(self, *, pk: uuid.UUID) -> bool:
association = self.get(pk=pk)
association.soft_delete()
return True

View File

@@ -0,0 +1,77 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
import logging
import uuid
from celery.result import AsyncResult
from django.utils.timezone import now
from hotpocket_backend.apps.bot.services import BotService
from hotpocket_backend.apps.saves.tasks import process_save
from .saves import SavesService
LOGGER = logging.getLogger(__name__)
class SaveProcessorService:
class SaveProcessorServiceError(Exception):
pass
class SkipReprocessing(SaveProcessorServiceError):
pass
def process(self, *, pk: uuid.UUID) -> bool:
result = True
try:
save = SavesService().get_for_processing(pk=pk)
assert save is not None, ('Could not fetch the save.')
title: str = save.url
description: str | None = None
is_netloc_banned = False
try:
bot_result = BotService().handle(url=save.url)
if bot_result.title is not None:
title = bot_result.title
description = bot_result.description
is_netloc_banned = bot_result.is_netloc_banned
except Exception as exception:
LOGGER.error(
'Unhandled exception when fetching save content: %s',
exception,
exc_info=exception,
)
if save.last_processed_at is not None:
raise self.SkipReprocessing(
'Not re-processing the save fetch error!',
) from exception
save.title = title
save.description = description
save.is_netloc_banned = is_netloc_banned
save.last_processed_at = now()
save.save()
except Exception as exception:
LOGGER.error(
'Unhandled exception: pk=`%s`: %s',
pk,
exception,
exc_info=exception,
)
result = False
return result
def schedule_process_save(self, *, pk: uuid.UUID) -> AsyncResult:
return process_save.apply_async(
kwargs={
'pk': pk,
},
)

View File

@@ -0,0 +1,104 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
import hashlib
import typing
import uuid
from django.db import models
from hotpocket_backend.apps.core.services import get_adapter
from hotpocket_backend.apps.saves.models import Save
from hotpocket_backend.apps.saves.types import PSaveAdapter
from hotpocket_soa.dto.saves import ImportedSaveIn, SaveIn, SavesQuery
class SavesService:
class SavesServiceError(Exception):
pass
class SaveNotFound(SavesServiceError):
pass
@property
def adapter(self) -> PSaveAdapter:
if hasattr(self, '_adapter') is False:
adapter_klass = get_adapter(
'SAVES_SAVE_ADAPTER',
'hotpocket_backend.apps.saves.adapters.basic:BasicSaveAdapter',
)
self._adapter = adapter_klass()
return self._adapter
def create(self,
*,
account_uuid: uuid.UUID,
save: SaveIn | ImportedSaveIn,
) -> Save:
key = hashlib.sha256(save.url.encode('utf-8')).hexdigest()
defaults = dict(
account_uuid=account_uuid,
key=key,
url=save.url,
)
save_object, created = Save.objects.get_or_create(
key=key,
deleted_at__isnull=True,
defaults=defaults,
)
if created is True:
save_object.is_netloc_banned = save.is_netloc_banned
if isinstance(save, ImportedSaveIn) is True:
save_object.title = save.title # type: ignore[union-attr]
save_object.save()
return save_object
def get(self, *, pk: uuid.UUID) -> Save:
try:
return Save.active_objects.get(pk=pk)
except Save.DoesNotExist as exception:
raise self.SaveNotFound(
f'Save not found: pk=`{pk}`',
) from exception
def search(self,
*,
filters: typing.Any,
offset: int = 0,
limit: int = 10,
order: str = '-created_at',
) -> models.QuerySet[Save]:
raise NotImplementedError('TODO')
def update(self, *, pk: uuid.UUID, save: SaveIn) -> Save:
raise NotImplementedError('TODO')
def delete(self, *, pk: uuid.UUID) -> bool:
raise NotImplementedError('TODO')
def search_associated_to_account(self,
*,
account_uuid: uuid.UUID,
query: SavesQuery,
offset: int = 0,
limit: int = 10,
order_by: str = '-associations__created_at',
) -> models.QuerySet[Save]:
result = Save.active_objects.\
filter(
associations__account_uuid=account_uuid,
associations__archived_at__isnull=True,
).\
order_by(order_by)
return result
def get_for_processing(self, *, pk: uuid.UUID) -> Save:
return self.adapter.get_for_processing(pk=pk)

View File

@@ -0,0 +1,15 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
import uuid
from celery import shared_task
from django import db
@shared_task
def process_save(*, pk: uuid.UUID):
from hotpocket_backend.apps.saves.services import SaveProcessorService
with db.transaction.atomic():
return SaveProcessorService().process(pk=pk)

View File

@@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import annotations
import typing
import uuid
from django.db import models
from hotpocket_backend.apps.saves.models import Save
from hotpocket_soa.dto.associations import AssociationsQuery
class PSaveAdapter(typing.Protocol):
def get_for_processing(self, *, pk: uuid.UUID) -> Save:
...
class PAssociationAdapter(typing.Protocol):
def get_search_filters(self, *, query: AssociationsQuery) -> list[models.Q]:
...