dagens_lunch/settings.py

@@ -89,6 +89,12 @@ CONCURRENT_REQUESTS_PER_DOMAIN = 8
 
 # Set settings whose default value is deprecated to a future-proof value
 TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+FEEDS = {
+    "items.json": {
+        "format": "json",
+        "encoding": "utf8"
+    }
+}
 FEED_EXPORT_ENCODING = "utf-8"
 
 LOG_ENABLED = False

dagens_lunch/spiders/heat.py

@@ -21,29 +21,15 @@ class HeatSpider(scrapy.Spider):
 
         for idx in range(1, 8):
             if not tree.find(f".//mandagratt{idx}rubrik") is None:
-                rubrik = tree.find(f".//mandagratt{idx}rubrik").text
+                self.monday.append({"rubrik": tree.find(f".//mandagratt{idx}rubrik").text, "text": tree.find(f".//mandagratt{idx}text").text})
-                text = tree.find(f".//mandagratt{idx}text").text
-                self.monday.append(f"{rubrik} {text}".strip())
-
             if not tree.find(f".//tisdagratt{idx}rubrik") is None:
-                rubrik = tree.find(f".//tisdagratt{idx}rubrik").text
+                self.tuesday.append({"rubrik": tree.find(f".//tisdagratt{idx}rubrik").text, "text": tree.find(f".//tisdagratt{idx}text").text})
-                text = tree.find(f".//tisdagratt{idx}text").text
-                self.tuesday.append(f"{rubrik} {text}".strip())
-
             if not tree.find(f".//onsdagratt{idx}rubrik") is None:
-                rubrik = tree.find(f".//onsdagratt{idx}rubrik").text
+                self.wednesday.append({"rubrik": tree.find(f".//onsdagratt{idx}rubrik").text, "text": tree.find(f".//onsdagratt{idx}text").text})
-                text = tree.find(f".//onsdagratt{idx}text").text
-                self.wednesday.append(f"{rubrik} {text}".strip())
-
             if not tree.find(f".//torsdagratt{idx}rubrik") is None:
-                rubrik = tree.find(f".//torsdagratt{idx}rubrik").text
+                self.thursday.append({"rubrik": tree.find(f".//torsdagratt{idx}rubrik").text, "text": tree.find(f".//torsdagratt{idx}text").text})
-                text = tree.find(f".//torsdagratt{idx}text").text
-                self.thursday.append(f"{rubrik} {text}".strip())
-
             if not tree.find(f".//fredagratt{idx}rubrik") is None:
-                rubrik = tree.find(f".//fredagratt{idx}rubrik").text
+                self.friday.append({"rubrik": tree.find(f".//fredagratt{idx}rubrik").text, "text": tree.find(f".//fredagratt{idx}text").text})
-                text = tree.find(f".//fredagratt{idx}text").text
-                self.friday.append(f"{rubrik} {text}".strip())
 
         dagens["week"] = self.week
         dagens["monday"] = self.monday
@@ -58,6 +44,7 @@ class HeatSpider(scrapy.Spider):
         url = ""
         dagens = DagensLunchHeat()
         dagens["place"] = "Heat Kopparlunden"
+        dagens["scraped_by"] = f"{self.__class__.__name__}"
         for js in scripts:
             if not "jQuery(function( $ ){" in js:
                 continue
@@ -67,7 +54,6 @@ class HeatSpider(scrapy.Spider):
                 if "url:\"https://castit.nu/xml" in line:
                     url = line[5:-2]
         yield scrapy.Request(url, callback=self.extract_xml, meta={"dagens": dagens})
-        dagens["scraped_by"] = f"{self.__class__.__name__}"
         dagens["scraped_at"] = f"{datetime.now().isoformat()}"
 
-        return dagens
+        return dagens

run_spiders.py

@@ -57,7 +57,6 @@ if __name__ == "__main__":
     _heat = None
     _max = None
     res = dagens_lunch_results()
-    dir.mkdir(parents=True, exist_ok=True)
     with file.open("a") as f:
         f.write("[\n")
         for idx, item in enumerate(res):