Updates to website issue

.devcontainer/devcontainer.json

@@ -11,7 +11,7 @@
 	"forwardPorts": [8000],
 
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "pip install -r requirements.txt && wget -O SIMPLE.sqlite https://raw.githubusercontent.com/SIMPLE-AstroDB/SIMPLE-binary/main/SIMPLE.sqlite && python -m simple_app.app_simple -d"
+	"postCreateCommand": "pip install -r requirements.txt && wget -O database.toml https://raw.githubusercontent.com/SIMPLE-AstroDB/SIMPLE-db/refs/heads/main/database.toml && wget -O SIMPLE.sqlite https://raw.githubusercontent.com/SIMPLE-AstroDB/SIMPLE-binary/main/SIMPLE.sqlite && python -m simple_app.app_simple -d"
 
 	// Configure tool-specific properties.
 	// "customizations": {},

.github/workflows/python-app.yml

@@ -22,8 +22,10 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        if [ -f requirements.txt ]; then pip install -Ur requirements.txt; fi
         wget -O SIMPLE.sqlite https://raw.githubusercontent.com/SIMPLE-AstroDB/SIMPLE-binary/main/SIMPLE.sqlite
+        wget -O database.toml https://raw.githubusercontent.com/SIMPLE-AstroDB/SIMPLE-db/refs/heads/main/database.toml
+        mkdir data
     - name: Test with pytest
       run: |
         pytest -v simple_app/tests/test_plots.py

.gitignore

@@ -3,6 +3,8 @@
 *.sqlite
 *.fits
 simple_app/tmp/user*
+*yaml
+*toml
 
 # Pycharm stuff
 .idea*
@@ -100,4 +102,4 @@ ENV/
 # Misc
 .DS_Store
 .vs/
-.vscode/
+.vscode/

cronjob.sh

@@ -0,0 +1,4 @@
+cd ~/ROOT
+wget -O SIMPLE.sqlite https://raw.githubusercontent.com/SIMPLE-AstroDB/SIMPLE-binary/main/SIMPLE.sqlite
+wget -O database.toml https://raw.githubusercontent.com/SIMPLE-AstroDB/SIMPLE-db/refs/heads/main/database.toml
+git pull

requirements.txt

@@ -1,6 +1,6 @@
 astrodbkit==2.5
 astropy==7.1.0
-bokeh==3.7.3
+bokeh==3.8.2
 Flask==3.1.1
 Flask-Cors==6.0.0
 Flask-WTF==1.2.2
@@ -9,9 +9,9 @@ multiprocess==0.70.17
 numpy==1.26.4
 pandas==2.0.3
 pytest==8.3.4
-requests==2.32.4
+requests==2.33.0
 specutils==2.2.0
 SQLAlchemy==2.0.38
 tqdm==4.67.1
-Werkzeug==3.1.4
-WTForms==3.2.1
+Werkzeug==3.1.5
+WTForms==3.2.1

simple_app/app_simple.py

@@ -170,12 +170,21 @@ def raw_query():
         except (ResourceClosedError, OperationalError, IndexError, SqliteWarning, BadSQLError, ProgrammingError):
             results = pd.DataFrame()
 
-        results = reference_handle(results, db_file, True)
+        if 'access_url' in results.columns:
+            url_links = [f'<a href="{url}" target="_blank">Link</a>' for url in results.access_url.values]
+            results.drop(columns=['access_url'], inplace=True)
+            download_col = '<a href="/write_sql_spectra" target="_blank">download</a>'
+            results.insert(1, download_col, url_links)
+
+        results = reference_handle(results, db_file)
+        res_len = len(results)
         stringed_results = one_df_query(results)
-        return render_template('raw_query.html', form=form, results=stringed_results, version_str=version_str)
+        return render_template('raw_query.html', form=form, results=stringed_results, query=query,
+                               res_len=res_len, version_str=version_str)
 
     else:
-        return render_template('raw_query.html', form=form, results=None, query='', version_str=version_str)
+        return render_template('raw_query.html', form=form, results=None, res_len=0, query='',
+                               version_str=version_str)
 
 
 @app_simple.route('/solo_result/<query>')
@@ -338,13 +347,8 @@ def create_spectra_files_for_download():
     results: pd.DataFrame = getattr(everything, 'spectra')
 
     # write all spectra for object to zipped file
-    zipped = write_spec_files(results.access_url.values)
-    if zipped is not None:
-        response = Response(zipped, mimetype='application/zip')
-        response = control_response(response, app_type='zip')
-        return response
-
-    abort(400, 'Could not download fits')
+    response = zip_spectra(results.access_url)
+    return response if response is not None else abort(400, 'Could not download fits')
 
 
 @app_simple.route('/write_multi_spectra', methods=['GET'])
@@ -360,13 +364,27 @@ def create_multi_spectra_files_for_download():
     spectra_df: pd.DataFrame = resultdict['Spectra']
 
     # write all spectra for object to zipped file
-    zipped = write_spec_files(spectra_df.access_url.values)
-    if zipped is not None:
-        response = Response(zipped, mimetype='application/zip')
-        response = control_response(response, app_type='zip')
-        return response
+    response = zip_spectra(spectra_df.access_url)
+    return response if response is not None else abort(400, 'Could not download fits')
+
+
+@app_simple.route('/write_sql_spectra', methods=['GET', 'POST'])
+def create_sql_spectra_files_for_download():
+    """
+    Downloads all spectra from a raw SQL query if an access_url column is present.
+    """
+    query = session.get('query')
+    db = SimpleDB(db_file)
+
+    # query database via sql
+    results: Optional[pd.DataFrame] = db.sql_query(query, fmt='pandas')
 
-    abort(400, 'Could not download fits')
+    if 'access_url' not in results.columns:
+        abort(400, 'No url column in SQL results')
+
+    # write all spectra in SQL result to zipped file
+    response = zip_spectra(results.access_url)
+    return response if response is not None else abort(400, 'Could not download fits')
 
 
 @app_simple.route('/write_filt', methods=['GET'])
@@ -474,6 +492,17 @@ def create_file_for_sql_download():
     return response
 
 
+@app_simple.route('/download_sqlite', methods=['GET'])
+def download_sqlite():
+    """
+    Downloads the SIMPLE.db file.
+    """
+    local_db_file = db_file.replace('sqlite:///', '')
+    directory = os.path.dirname(local_db_file) or '..'  # code runs in simple_app dir, we keep the binary one dir up
+    filename = os.path.basename(local_db_file)
+    return send_from_directory(directory, filename, as_attachment=True)
+
+
 args, db_file, photometric_filters, all_results, all_results_full, version_str, \
     all_photometry, all_bands, all_parallaxes, all_spectral_types = main_utils()
 night_sky_theme, js_callbacks = main_plots()

simple_app/simports.py

@@ -12,7 +12,7 @@
 from sqlite3 import Warning as SqliteWarning  # errors from sqlite
 from time import localtime, strftime  # time stuff for naming files
 from typing import Dict, Generator, List, Optional, Tuple, Union  # type hinting (good in IDEs)
-from urllib.parse import quote  # handling strings into url friendly form
+from urllib.parse import quote, unquote, urlparse  # handling strings into url friendly form
 from zipfile import ZipFile  # zipping files together
 
 import astropy.units as u  # units
@@ -67,3 +67,7 @@
 from tqdm import tqdm  # progress bars
 from werkzeug.exceptions import HTTPException  # underlying http
 from wtforms import StringField, SubmitField, TextAreaField, ValidationError  # web forms
+import tomllib
+with open("database.toml", "rb") as f:
+    settings = tomllib.load(f)
+    REFERENCE_TABLES = settings['lookup_tables']