added conda environment file

environment.yml

@@ -0,0 +1,78 @@
+name: crawlers
+channels:
+  - defaults
+  - https://repo.anaconda.com/pkgs/main
+  - https://repo.anaconda.com/pkgs/r
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - appdirs=1.4.4=pyhd3eb1b0_0
+  - attrs=24.3.0=py312h06a4308_0
+  - automat=20.2.0=py_0
+  - bcrypt=3.2.0=py312h5eee18b_2
+  - brotli-python=1.0.9=py312h6a678d5_9
+  - bzip2=1.0.8=h5eee18b_6
+  - ca-certificates=2024.12.31=h06a4308_0
+  - certifi=2025.1.31=py312h06a4308_0
+  - cffi=1.17.1=py312h1fdaa30_1
+  - charset-normalizer=3.3.2=pyhd3eb1b0_0
+  - constantly=23.10.4=py312h06a4308_0
+  - cryptography=43.0.3=py312h7825ff9_1
+  - cssselect=1.2.0=py312h06a4308_0
+  - defusedxml=0.7.1=pyhd3eb1b0_0
+  - expat=2.6.4=h6a678d5_0
+  - filelock=3.13.1=py312h06a4308_0
+  - hyperlink=21.0.0=pyhd3eb1b0_0
+  - icu=73.1=h6a678d5_0
+  - idna=3.7=py312h06a4308_0
+  - incremental=22.10.0=pyhd3eb1b0_0
+  - itemadapter=0.3.0=pyhd3eb1b0_0
+  - itemloaders=1.1.0=py312h06a4308_0
+  - jmespath=1.0.1=py312h06a4308_0
+  - ld_impl_linux-64=2.40=h12ee557_0
+  - libffi=3.4.4=h6a678d5_1
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - libuuid=1.41.5=h5eee18b_0
+  - libxml2=2.13.5=hfdd30dd_0
+  - libxslt=1.1.41=h097e994_0
+  - lxml=5.3.0=py312h57af460_1
+  - ncurses=6.4=h6a678d5_0
+  - openssl=3.0.15=h5eee18b_0
+  - packaging=24.2=py312h06a4308_0
+  - parsel=1.8.1=py312h06a4308_0
+  - pip=25.0=py312h06a4308_0
+  - protego=0.1.16=py_0
+  - pyasn1=0.4.8=pyhd3eb1b0_0
+  - pyasn1-modules=0.2.8=py_0
+  - pycparser=2.21=pyhd3eb1b0_0
+  - pydispatcher=2.0.5=py312h06a4308_3
+  - pyopenssl=24.2.1=py312h06a4308_0
+  - pysocks=1.7.1=py312h06a4308_0
+  - python=3.12.9=h5148396_0
+  - queuelib=1.6.2=py312h06a4308_0
+  - readline=8.2=h5eee18b_0
+  - requests=2.32.3=py312h06a4308_1
+  - requests-file=1.5.1=pyhd3eb1b0_0
+  - scrapy=2.12.0=py312h06a4308_0
+  - service_identity=18.1.0=pyhd3eb1b0_1
+  - setuptools=75.8.0=py312h06a4308_0
+  - six=1.16.0=pyhd3eb1b0_1
+  - sqlite=3.45.3=h5eee18b_0
+  - tk=8.6.14=h39e8969_0
+  - tldextract=5.1.2=py312h06a4308_0
+  - twisted=23.10.0=py312h06a4308_0
+  - typing_extensions=4.12.2=py312h06a4308_0
+  - tzdata=2025a=h04d1e81_0
+  - urllib3=2.3.0=py312h06a4308_0
+  - w3lib=1.21.0=pyhd3eb1b0_0
+  - wheel=0.45.1=py312h06a4308_0
+  - xz=5.6.4=h5eee18b_1
+  - zlib=1.2.13=h5eee18b_1
+  - zope=1.0=py312h06a4308_1
+  - zope.interface=7.1.1=py312h5eee18b_0
+  - pip:
+      - chompjs==1.3.0
+      - pypdf2==3.0.1
+prefix: /home/riisen/utveckling/miniconda3/envs/crawlers

scripts/bash_alias.sh

@@ -0,0 +1,4 @@
+alias dagens_lunch_fetch="${HOME}/scripts/crawlers/dagens_lunch.sh"
+alias lunch_malmens="cat ${HOME}/utveckling/crawlers/dagens_lunch/latest/malmens.json | python -m json.tool | jq"
+alias lunch_heat="cat ${HOME}/utveckling/crawlers/dagens_lunch/latest/heat.json | python -m json.tool | jq"
+alias lunch_valsaren="cat ${HOME}/utveckling/crawlers/dagens_lunch/latest/valsaren.json | python -m json.tool | jq"

scripts/run_spiders.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+source ${HOME}/utveckling/miniconda3/bin/activate
+conda activate crawlers
+cd ${HOME}/utveckling/crawlers/dagens_lunch
+echo "removing old json files.."
+rm latest/*.json
+
+echo "getting menu from valsaren.."
+scrapy crawl valsaren -o latest/valsaren.json > /dev/null 2>&1
+ 
+echo "getting menu from heat.."
+scrapy crawl heat -o latest/heat.json > /dev/null 2>&1
+ 
+echo "getting menu from malmens.."
+scrapy crawl malmens -o latest/malmens.json > /dev/null 2>&1