fix: Wire bcp, std, fyi index creation (#10706)

kesara · web-flow · commit 1689b88859e7 · 2026-04-14T01:06:34.000-03:00
* style: Ruff Ruff

* fix: Wire bcp and std index creation tasks

* fix: Use single task and add fyi index creation

* fix: Log index creation errors and continue
diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py
@@ -32,7 +32,9 @@
     EmailPersonSerializer,
     RfcWithAuthorsSerializer,
     DraftWithAuthorsSerializer,
-    NotificationAckSerializer, RfcPubSerializer, RfcFileSerializer,
+    NotificationAckSerializer,
+    RfcPubSerializer,
+    RfcFileSerializer,
     EditableRfcSerializer,
 )
 from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent
@@ -344,9 +346,10 @@ def post(self, request):
 
 class RfcAuthorViewSet(viewsets.ReadOnlyModelViewSet):
     """ViewSet for RfcAuthor model
-    
+
     Router needs to provide rfc_number as a kwarg
     """
+
     api_key_endpoint = "ietf.api.views_rpc"
 
     queryset = RfcAuthor.objects.all()
@@ -407,7 +410,7 @@ class RfcPubFilesView(APIView):
 
     def _fs_destination(self, filename: str | Path) -> Path:
         """Destination for an uploaded RFC file in the filesystem
-        
+
         Strips any path components in filename and returns an absolute Path.
         """
         rfc_path = Path(settings.RFC_PATH)
@@ -419,7 +422,7 @@ def _fs_destination(self, filename: str | Path) -> Path:
 
     def _blob_destination(self, filename: str | Path) -> str:
         """Destination name for an uploaded RFC file in the blob store
-        
+
         Strips any path components in filename and returns an absolute Path.
         """
         filename = Path(filename)  # could potentially have directory components
@@ -472,9 +475,7 @@ def post(self, request):
                         code="files-exist",
                     )
             for possible_existing_blob in possible_rfc_blobs:
-                if exists_in_storage(
-                    kind=blob_kind, name=possible_existing_blob
-                ):
+                if exists_in_storage(kind=blob_kind, name=possible_existing_blob):
                     raise Conflict(
                         "Blob(s) already exist for this RFC",
                         code="blobs-exist",
@@ -523,7 +524,9 @@ def post(self, request):
 
         # Trigger red precomputer
         needs_updating = [rfc.rfc_number]
-        for rel in rfc.relateddocument_set.filter(relationship_id__in=["obs","updates"]):
+        for rel in rfc.relateddocument_set.filter(
+            relationship_id__in=["obs", "updates"]
+        ):
             needs_updating.append(rel.target.rfc_number)
         trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating))
         # Trigger search index update
@@ -540,7 +543,7 @@ class RfcIndexView(APIView):
     @extend_schema(
         operation_id="refresh_rfc_index",
         summary="Refresh rfc-index files",
-        description="Requests creation of rfc-index.xml and rfc-index.txt files",
+        description="Requests creation of various index files.",
         responses={202: None},
         request=None,
     )
diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py
@@ -18,7 +18,13 @@
 from ietf.sync import iana
 from ietf.sync import rfceditor
 from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue
-from ietf.sync.rfcindex import create_rfc_txt_index, create_rfc_xml_index
+from ietf.sync.rfcindex import (
+    create_bcp_txt_index,
+    create_fyi_txt_index,
+    create_rfc_txt_index,
+    create_rfc_xml_index,
+    create_std_txt_index,
+)
 from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper
 from ietf.utils import log
 from ietf.utils.timezone import date_today
@@ -27,13 +33,13 @@
 @shared_task
 def rfc_editor_index_update_task(full_index=False):
     """Update metadata from the RFC index
-    
+
     Default is to examine only changes in the past 365 days. Call with full_index=True to update
     the full RFC index.
-    
+
     According to comments on the original script, a year's worth took about 20s on production as of
     August 2022
-    
+
     The original rfc-editor-index-update script had a long-disabled provision for running the
     rebuild_reference_relations scripts after the update. That has not been brought over
     at all because it should be implemented as its own task if it is needed.
@@ -51,7 +57,7 @@ def rfc_editor_index_update_task(full_index=False):
             timeout=30,  # seconds
         )
     except requests.Timeout as exc:
-        log.log(f'GET request timed out retrieving RFC editor index: {exc}')
+        log.log(f"GET request timed out retrieving RFC editor index: {exc}")
         return  # failed
     rfc_index_xml = response.text
     index_data = rfceditor.parse_index(io.StringIO(rfc_index_xml))
@@ -61,9 +67,9 @@ def rfc_editor_index_update_task(full_index=False):
             timeout=30,  # seconds
         )
     except requests.Timeout as exc:
-        log.log(f'GET request timed out retrieving RFC editor errata: {exc}')
+        log.log(f"GET request timed out retrieving RFC editor errata: {exc}")
         return  # failed
-    errata_data = response.json()   
+    errata_data = response.json()
     if len(index_data) < rfceditor.MIN_INDEX_RESULTS:
         log.log("Not enough index entries, only %s" % len(index_data))
         return  # failed
@@ -96,15 +102,15 @@ def rfc_editor_queue_updates_task():
     drafts, warnings = parse_queue(io.StringIO(response.text))
     for w in warnings:
         log.log(f"Warning: {w}")
-    
+
     if len(drafts) < MIN_QUEUE_RESULTS:
         log.log("Not enough results, only %s" % len(drafts))
         return  # failed
-    
+
     changed, warnings = update_drafts_from_queue(drafts)
     for w in warnings:
         log.log(f"Warning: {w}")
-    
+
     for c in changed:
         log.log(f"Updated {c}")
 
@@ -120,9 +126,11 @@ def iana_changes_update_task():
     MAX_INTERVAL_ACCEPTED_BY_IANA = datetime.timedelta(hours=23)
 
     start = (
-        timezone.now() 
-        - datetime.timedelta(hours=23) 
-        + datetime.timedelta(seconds=CLOCK_SKEW_COMPENSATION,)
+        timezone.now()
+        - datetime.timedelta(hours=23)
+        + datetime.timedelta(
+            seconds=CLOCK_SKEW_COMPENSATION,
+        )
     )
     end = start + datetime.timedelta(hours=23)
 
@@ -133,7 +141,9 @@ def iana_changes_update_task():
         # requests if necessary
 
         text = iana.fetch_changes_json(
-            settings.IANA_SYNC_CHANGES_URL, t, min(end, t + MAX_INTERVAL_ACCEPTED_BY_IANA)
+            settings.IANA_SYNC_CHANGES_URL,
+            t,
+            min(end, t + MAX_INTERVAL_ACCEPTED_BY_IANA),
         )
         log.log(f"Retrieved the JSON: {text}")
 
@@ -159,9 +169,9 @@ def iana_protocols_update_task():
     # "this needs to be the date where this tool is first deployed" in the original
     # iana-protocols-updates script)"
     rfc_must_published_later_than = datetime.datetime(
-        2012, 
-        11, 
-        26, 
+        2012,
+        11,
+        26,
         tzinfo=datetime.UTC,
     )
 
@@ -171,17 +181,17 @@ def iana_protocols_update_task():
             timeout=30,
         )
     except requests.Timeout as exc:
-        log.log(f'GET request timed out retrieving IANA protocols page: {exc}')
+        log.log(f"GET request timed out retrieving IANA protocols page: {exc}")
         return
 
     rfc_numbers = iana.parse_protocol_page(response.text)
 
     def batched(l, n):
         """Split list l up in batches of max size n.
-        
+
         For Python 3.12 or later, replace this with itertools.batched()
         """
-        return (l[i:i + n] for i in range(0, len(l), n))
+        return (l[i : i + n] for i in range(0, len(l), n))
 
     for batch in batched(rfc_numbers, 100):
         updated = iana.update_rfc_log_from_protocol_page(
@@ -192,6 +202,7 @@ def batched(l, n):
         for d in updated:
             log.log("Added history entry for %s" % d.display_name())
 
+
 @shared_task
 def fix_subseries_docevents_task():
     """Repairs DocEvents related to bugs around removing docs from subseries
@@ -233,6 +244,7 @@ def fix_subseries_docevents_task():
             time=obsoleting_time
         )
 
+
 @shared_task
 def rsync_rfcs_from_rfceditor_task(rfc_numbers: list[int]):
     log.log(f"Rsyncing rfcs from rfc-editor: {rfc_numbers}")
@@ -277,6 +289,32 @@ def load_rfcs_into_blobdb_task(start: int, end: int):
 
 @shared_task
 def create_rfc_index_task():
-    create_rfc_txt_index()
-    create_rfc_xml_index()
+    try:
+        create_rfc_txt_index()
+    except Exception as e:
+        log.log(f"Error: failure in creating rfc-index.txt. {e}")
+        pass
 
+    try:
+        create_rfc_xml_index()
+    except Exception as e:
+        log.log(f"Error: failure in creating rfc-index.xml. {e}")
+        pass
+
+    try:
+        create_bcp_txt_index()
+    except Exception as e:
+        log.log(f"Error: failure in creating bcp-index.txt. {e}")
+        pass
+
+    try:
+        create_std_txt_index()
+    except Exception as e:
+        log.log(f"Error: failure in creating std-index.txt. {e}")
+        pass
+
+    try:
+        create_fyi_txt_index()
+    except Exception as e:
+        log.log(f"Error: failure in creating fyi-index.txt. {e}")
+        pass
