[pull] master from kevoreilly:master

conf/default/processing.conf.default

@@ -160,6 +160,13 @@ remove_empty = yes
 key = a0283a2c3d55728300d064874239b5346fb991317e8449fe43c902879d758088
 do_file_lookup = yes
 do_url_lookup = yes
+# Use cached results from MongoDB if available
+cache_default = no
+cache_static = no
+cache_file = no
+cache_dropped = no
+cache_cape = no
+cache_procdump = no
 urlscrub = (^http:\/\/serw\.clicksor\.com\/redir\.php\?url=|&InjectedParam=.+$)
 
 # Since Suricata 8, socket mode is deprecated.

lib/cuckoo/common/integrations/file_extra_info.py

@@ -156,6 +156,7 @@ def static_file_info(
     destination_folder: str,
     results: dict,
     duplicated: DuplicatesType,
+    category: str = "files",
 ):
     size_mb = int(path_get_size(file_path) / (1024 * 1024))
     if size_mb > int(processing_conf.CAPE.max_file_size):
@@ -266,7 +267,7 @@ def static_file_info(
 
         # ToDo we need url support
         if HAVE_VIRUSTOTAL and processing_conf.virustotal.enabled and "virustotal" not in data_dictionary:
-            vt_details = vt_lookup("file", file_path, results)
+            vt_details = vt_lookup("file", file_path, results, file_category=category)
             if vt_details:
                 data_dictionary["virustotal"] = vt_details
 

lib/cuckoo/common/integrations/virustotal.py

@@ -14,6 +14,7 @@
 from lib.cuckoo.common.objects import File
 from lib.cuckoo.common.path_utils import path_exists
 from lib.cuckoo.common.utils import add_family_detection
+from dev_utils.mongodb import mongo_find_one
 
 try:
     import re2 as re
@@ -36,6 +37,15 @@
 timeout = int(processing_conf.virustotal.timeout)
 remove_empty = processing_conf.virustotal.remove_empty
 
+cache_default = processing_conf.virustotal.get("cache_default", False)
+VT_CACHE_MAP = {
+    "static": processing_conf.virustotal.get("cache_static", cache_default),
+    "file": processing_conf.virustotal.get("cache_file", cache_default),
+    "dropped": processing_conf.virustotal.get("cache_dropped", cache_default),
+    "cape": processing_conf.virustotal.get("cache_cape", cache_default),
+    "procdump": processing_conf.virustotal.get("cache_procdump", cache_default),
+}
+
 headers = {"x-apikey": key}
 
 """
@@ -185,7 +195,7 @@ def get_vt_consensus(namelist: list):
     return ""
 
 
-def vt_lookup(category: str, target: str, results: dict = {}, on_demand: bool = False):
+def vt_lookup(category: str, target: str, results: dict = {}, on_demand: bool = False, file_category: str = ""):
     if not processing_conf.virustotal.enabled or processing_conf.virustotal.get("on_demand", False) and not on_demand:
         return {}
     if category not in ("file", "url"):
@@ -199,6 +209,18 @@ def vt_lookup(category: str, target: str, results: dict = {}, on_demand: bool =
             return {"error": True, "msg": "File doesn't exist"}
 
         sha256 = target if len(target) == 64 else File(target).get_sha256()
+
+        if file_category:
+            cache_setting = VT_CACHE_MAP.get(file_category.lower(), cache_default)
+
+            if cache_setting:
+                try:
+                    db_file = mongo_find_one("files", {"sha256": sha256})
+                    if db_file and "virustotal" in db_file:
+                        return db_file["virustotal"]
+                except Exception as e:
+                    log.error("Error checking VT cache: %s", e)
+
         url = VIRUSTOTAL_FILE_URL.format(id=sha256)
 
     elif category == "url":

modules/processing/CAPE.py

@@ -43,6 +43,12 @@
 integrations_conf = Config("integrations")
 externalservices_conf = Config("externalservices")
 
+HAVE_VIRUSTOTAL = False
+if processing_conf.virustotal.enabled and not processing_conf.virustotal.on_demand:
+    with suppress(ImportError):
+        from lib.cuckoo.common.integrations.virustotal import vt_lookup
+        HAVE_VIRUSTOTAL = True
+
 HAVE_FLARE_CAPA = False
 # required to not load not enabled dependencies
 if integrations_conf.flare_capa.enabled and not integrations_conf.flare_capa.on_demand:
@@ -208,6 +214,11 @@ def process_file(self, file_path, append_file, metadata: dict, *, category: str,
                     cached = True
                     if yara_match and options_match:
                         run_static = False
+                        if HAVE_VIRUSTOTAL:
+                            # We might want to refresh VT based on cache policy
+                            vt_details = vt_lookup("file", sha256, self.results, file_category=category)
+                            if vt_details:
+                                file_info["virustotal"] = vt_details
                     else:
                         # We need to re-run static/tools
                         run_static = True
@@ -267,6 +278,7 @@ def process_file(self, file_path, append_file, metadata: dict, *, category: str,
                 self.self_extracted,
                 self.results,
                 duplicated,
+                category=category,
             )
 
         type_string, append_file = self._metadata_processing(metadata, file_info, append_file)