From 17223b29adf79c5c3eb3122d572a646d62ac1a7f Mon Sep 17 00:00:00 2001 From: jkrick Date: Wed, 25 Feb 2026 13:54:43 -0800 Subject: [PATCH 1/3] Refine Euclid heading cleanup and revert SIA tutorial edits --- tutorials/euclid/1_Euclid_intro_MER_images.md | 44 +++++++++---------- .../euclid/4_Euclid_intro_PHZ_catalog.md | 20 ++++----- .../euclid/5_Euclid_intro_SPE_catalog.md | 2 +- tutorials/euclid/Euclid_ERO.md | 24 +++++----- .../OpenUniverse2024Preview_Firefly.md | 16 +++---- .../roman_hlss_number_density.md | 6 +-- .../Parallelize_Convolution.md | 12 ++--- .../techniques-and-tools/SEDs_in_Firefly.md | 10 ++--- tutorials/wise/NEOWISE_light_curve_demo.md | 10 ++--- tutorials/wise/wise-allwise-catalog-demo.md | 10 ++--- 10 files changed, 77 insertions(+), 77 deletions(-) diff --git a/tutorials/euclid/1_Euclid_intro_MER_images.md b/tutorials/euclid/1_Euclid_intro_MER_images.md index 4a02b0a8..2b4e5a05 100644 --- a/tutorials/euclid/1_Euclid_intro_MER_images.md +++ b/tutorials/euclid/1_Euclid_intro_MER_images.md @@ -226,20 +226,20 @@ science_images['filters'][science_images['filters']== 'VIS_VIS'] = "VIS" science_images['filters'] ``` -## The image above is very large, so let's cut out a smaller image to inspect these data. +## 4. Define cutout parameters for a smaller region of interest ```{code-cell} ipython3 ######################## User defined section ############################ -## How large do you want the image cutout to be? +# Set the image cutout size im_cutout = 1.0 * u.arcmin -## What is the center of the cutout? -## For now choosing a random location on the image -## because the star itself is saturated +# Set the cutout center coordinates +# For now choose a random location on the image +# because the star itself is saturated ra = 273.8667 dec = 64.525 -## Bright star position +# Bright star position # ra = 273.474451 # dec = 64.397273 @@ -247,37 +247,37 @@ coords_cutout = SkyCoord(ra, dec, unit='deg', frame='icrs') ########################################################################## -## Iterate through each filter +# Iterate through each filter cutout_list = [] for url in urls: - ## Use fsspec to interact with the fits file without downloading the full file + # Use fsspec to interact with the fits file without downloading the full file hdu = fits.open(url, use_fsspec=True) print(f"Opened {url}") - ## Store the header + # Store the header header = hdu[0].header - ## Read in the cutout of the image that you want + # Read in the cutout of the image that you want cutout_data = Cutout2D(hdu[0].section, position=coords_cutout, size=im_cutout, wcs=WCS(hdu[0].header)) - ## Close the file + # Close the file # hdu.close() - ## Define a new fits file based on this smaller cutout, with accurate WCS based on the cutout size + # Define a new fits file based on this smaller cutout, with accurate WCS based on the cutout size new_hdu = fits.PrimaryHDU(data=cutout_data.data, header=header) new_hdu.header.update(cutout_data.wcs.to_header()) - ## Append the cutout to the list + # Append the cutout to the list cutout_list.append(new_hdu) -## Combine all cutouts into a single HDUList and display information +# Combine all cutouts into a single HDUList and display information final_hdulist = fits.HDUList(cutout_list) final_hdulist.info() ``` -## 3. Visualize multiwavelength Euclid Q1 MER cutouts +## 5. Visualize multiwavelength Euclid Q1 MER cutouts We need to determine the number of images for the grid layout, and then plot each cutout. @@ -297,7 +297,7 @@ for idx, (ax, filt) in enumerate(zip(axes, science_images['filters'])): ax.set_ylabel('Dec') ax.text(0.05, 0.05, filt, color='white', fontsize=14, transform=ax.transAxes, va='bottom', ha='left') -## Remove empty subplots if any +# Remove empty subplots if any for ax in axes[num_images:]: fig.delaxes(ax) @@ -305,7 +305,7 @@ plt.tight_layout() plt.show() ``` -## 4. Use the Python package sep to identify and measure sources in the Euclid Q1 MER cutouts +## 6. Identify and measure sources in Euclid Q1 MER cutouts with sep First we list all the filters so you can choose which cutout you want to extract sources on. We will choose VIS. @@ -354,13 +354,13 @@ data_sub = img2 - bkg ```{code-cell} ipython3 ######################## User defined section ############################ -## Sigma threshold to consider this a detection above the global RMS +# Sigma threshold to consider this a detection above the global RMS threshold= 3 -## Minimum number of pixels required for an object. Default is 5. +# Minimum number of pixels required for an object. Default is 5. minarea_0=2 -## Minimum contrast ratio used for object deblending. Default is 0.005. To entirely disable deblending, set to 1.0. +# Minimum contrast ratio used for object deblending. Default is 0.005. To entirely disable deblending, set to 1.0. deblend_cont_0= 0.005 flux_threshold= 0.01 @@ -372,7 +372,7 @@ sources_thr = sources[sources['flux'] > flux_threshold] print("Found", len(sources_thr), "objects above flux threshold") ``` -## Lets have a look at the objects that were detected with sep in the cutout +## 7. Review detected sources on the VIS cutout We plot the VIS cutout with the sources detected overplotted with a red ellipse @@ -382,7 +382,7 @@ fig, ax = plt.subplots() m, s = np.mean(data_sub), np.std(data_sub) im = ax.imshow(data_sub, cmap='gray', origin='lower', norm=ImageNormalize(img2, interval=ZScaleInterval(), stretch=SquaredStretch())) -## Plot an ellipse for each object detected with sep +# Plot an ellipse for each object detected with sep for i in range(len(sources_thr)): e = Ellipse(xy=(sources_thr['x'][i], sources_thr['y'][i]), diff --git a/tutorials/euclid/4_Euclid_intro_PHZ_catalog.md b/tutorials/euclid/4_Euclid_intro_PHZ_catalog.md index 1b4da925..01765897 100644 --- a/tutorials/euclid/4_Euclid_intro_PHZ_catalog.md +++ b/tutorials/euclid/4_Euclid_intro_PHZ_catalog.md @@ -186,10 +186,10 @@ Search based on ``tileID``: ```{code-cell} ipython3 ######################## User defined section ############################ -## How large do you want the image cutout to be? +# Set the image cutout size im_cutout= 5 * u.arcmin -## What is the center of the cutout? +# Set the center of the cutout ra_cutout = 267.8 dec_cutout = 66 @@ -215,7 +215,7 @@ adql = ("SELECT DISTINCT mer.object_id, mer.ra, mer.dec, " "AND phz.phz_median BETWEEN 1.4 AND 1.6") -## Use TAP with this ADQL string +# Use TAP with this ADQL string result_galaxies = Irsa.query_tap(adql).to_table() result_galaxies[:5] ``` @@ -235,13 +235,13 @@ Once the bug is fixed, we plan to update the code in this notebook and simplify Due to the large field of view of the MER mosaic, let's cut out a smaller section (5'x5') of the MER mosaic to inspect the image. ```{code-cell} ipython3 -## Use fsspec to interact with the fits file without downloading the full file +# Use fsspec to interact with the fits file without downloading the full file hdu = fits.open(filename, use_fsspec=True) -## Store the header +# Store the header header = hdu[0].header -## Read in the cutout of the image that you want +# Read in the cutout of the image that you want cutout_image = Cutout2D(hdu[0].section, position=coords_cutout, size=im_cutout, wcs=WCS(header)) ``` @@ -275,7 +275,7 @@ plt.scatter(result_galaxies['ra'], result_galaxies['dec'], s=36, facecolors='non _ = plt.title('Galaxies between z = 1.4 and 1.6') ``` -## 5. Pull the spectra on the top brightest source based on object ID +## 5. Pull spectra for one of the brightest sources by object ID ```{code-cell} ipython3 result_galaxies.sort(keys='flux_h_unif', reverse=True) @@ -299,7 +299,7 @@ We will use TAP and an ASQL query to find the spectral data for this particular ```{code-cell} ipython3 adql_object = f"SELECT * FROM {table_1dspectra} WHERE objectid = {obj_id}" -## Pull the data on this particular galaxy +# Pull the data on this particular galaxy result_spectra = Irsa.query_tap(adql_object).to_table() result_spectra ``` @@ -340,7 +340,7 @@ result_galaxies[index] ``` ```{code-cell} ipython3 -## How large do you want the image cutout to be? +# Set the image cutout size for the selected galaxy size_galaxy_cutout = 2.0 * u.arcsec ``` @@ -369,7 +369,7 @@ ax.imshow(cutout_galaxy.data, cmap='gray', origin='lower', norm=ImageNormalize(cutout_galaxy.data, interval=PercentileInterval(99.9), stretch=AsinhStretch())) ``` -## 6. Load the image on Firefly to be able to interact with the data directly +## 6. Load the image in Firefly for interactive exploration +++ diff --git a/tutorials/euclid/5_Euclid_intro_SPE_catalog.md b/tutorials/euclid/5_Euclid_intro_SPE_catalog.md index 94108edc..bbe5dd62 100644 --- a/tutorials/euclid/5_Euclid_intro_SPE_catalog.md +++ b/tutorials/euclid/5_Euclid_intro_SPE_catalog.md @@ -172,7 +172,7 @@ Irsa.list_columns(catalog=table_1dspectra, full=True) columns_info ``` -## Find some objects with spectra in our tileID +## 3. Find some objects with spectra in our tileID We specify the following conditions on our search: - Signal to noise ratio column (_gf = gaussian fit) should be greater than 5 diff --git a/tutorials/euclid/Euclid_ERO.md b/tutorials/euclid/Euclid_ERO.md index 8b3bb309..9f544eab 100644 --- a/tutorials/euclid/Euclid_ERO.md +++ b/tutorials/euclid/Euclid_ERO.md @@ -97,7 +97,7 @@ import matplotlib as mpl Next, we define some parameters for `Matplotlib` plotting. ```{code-cell} ipython3 -## Plotting stuff +## 1. Plotting stuff mpl.rcParams['font.size'] = 14 mpl.rcParams['axes.labelpad'] = 7 mpl.rcParams['xtick.major.pad'] = 7 @@ -123,7 +123,7 @@ mpl.rcParams['hatch.linewidth'] = 1 def_cols = plt.rcParams['axes.prop_cycle'].by_key()['color'] ``` -## Setting up the Environment +## 2. Setting up the Environment Next, we set up the environment. This includes * setting up an output data directory (will be created if it does not exist) @@ -148,7 +148,7 @@ cutout_size = 1.5 * u.arcmin # cutout size coord = SkyCoord.from_name('NGC 6397') ``` -## Search Euclid ERO Images +## 3. Search Euclid ERO Images Now, we search for the Euclid ERO images using the `astroquery` package. Note that the Euclid ERO images are no in the cloud currently, but we access them directly from IRSA using IRSA's *Simple Image Access* (SIA) methods. @@ -227,7 +227,7 @@ Let's check out the summary table that we have created. We see that we have all summary_table ``` -## Create Cutout Images +## 4. Create Cutout Images Now that we have a list of data products, we can create the cutouts. This is important as the full Euclid ERO images would be too large to run extraction and photometry software on them (they would simply fail due to memory issues). @@ -265,7 +265,7 @@ for ii,filt in tqdm(enumerate(filters)): hdu.header["FILTER"] = filt.upper() hdulcutout.append(hdu) -## Save the HDUL cube: +## 5. Save the HDUL cube: hdulcutout.writeto("./data/euclid_images_test.fits", overwrite=True) ``` @@ -294,7 +294,7 @@ for ii,filt in enumerate(filters): plt.show() ``` -## Extract Sources and Measure their Photometry on the VIS Image +## 6. Extract Sources and Measure their Photometry on the VIS Image Now that we have the images in memory (and on disk - but we do not need them, yet), we can measure the fluxes of the individual stars. Our simple photometry pipeline has different parts: @@ -308,7 +308,7 @@ Our simple photometry pipeline has different parts: We start by extracting the sources using `sep`. We first isolate the data that we want to look at (the VIS image only). ```{code-cell} ipython3 -## Get Data (this will be replaced later) +## 7. Get Data (this will be replaced later) img = hdulcutout["VIS_SCIENCE"].data hdr = hdulcutout["VIS_SCIENCE"].header img[img == 0] = np.nan @@ -378,7 +378,7 @@ resimage = psfphot.make_residual_image(data = img-median, psf_shape = (9, 9)) We now want to add the best-fit coordinates (R.A. and Decl.) to the VIS photometry catalog. For this, we have to convert the image coordinates into sky coordinates using the WCS information. We will need these coordinates because we want to use them as positional priors for the photometry measurement on the NISP images. ```{code-cell} ipython3 -## Add coordinates to catalog +## 8. Add coordinates to catalog wcs1 = WCS(hdr) # VIS radec = wcs1.all_pix2world(phot["x_fit"],phot["y_fit"],0) phot["ra_fit"] = radec[0] @@ -424,7 +424,7 @@ ax1.set_yscale('log') plt.show() ``` -## Measure the Photometry on the NISP Images +## 9. Measure the Photometry on the NISP Images We now have the photometry and the position of sources on the VIS image. We can now proceed with similar steps on the NISP images. Because the NISP PSF and pixel scale are larger that those of the VIS images, we utilize the advantage of position prior-based forced photometry. For this, we use the positions of the VIS measurements and perform PSF fitting on the NISP image using these priors. @@ -508,7 +508,7 @@ ax2.plot(phot2["x_fit"], phot2["y_fit"] , "o", markersize=8 , markeredgecolor="r plt.show() ``` -## Load Gaia Catalog +## 10. Load Gaia Catalog We now load the Gaia sources at the location of the globular clusters. The goal is to compare the photometry of Gaia to the one derived above for the Euclid VIS and NISP images. This is scientifically useful, for example we can compute the colors of the stars in the Gaia optical bands and the Euclid near-IR bands. To search for Gaia sources, we use `astroquery` again. @@ -563,7 +563,7 @@ ax2.set_title("NISP") plt.show() ``` -## Match the Gaia Catalog to the VIS and NISP Catalogs +## 11. Match the Gaia Catalog to the VIS and NISP Catalogs Now, we match the Gaia source positions to the extracted sources in the VIS and NISP images. @@ -628,7 +628,7 @@ ax1.set_ylabel("I$_E$ [mag]") plt.show() ``` -## Visualization with Firefly +## 12. Visualization with Firefly At the end of this Notebook, we demonstrate how we can visualize the images and catalogs created above in `Firefly`. diff --git a/tutorials/simulated-data/OpenUniverse2024Preview_Firefly.md b/tutorials/simulated-data/OpenUniverse2024Preview_Firefly.md index 1868c8af..a4058529 100644 --- a/tutorials/simulated-data/OpenUniverse2024Preview_Firefly.md +++ b/tutorials/simulated-data/OpenUniverse2024Preview_Firefly.md @@ -86,7 +86,7 @@ from reproject import reproject_interp from io import BytesIO ``` -## Learn where the OpenUniverse2024 data are hosted in the cloud. +## 1. Learn where the OpenUniverse2024 data are hosted in the cloud. The OpenUniverse2024 data preview is hosted in the cloud via Amazon Web Services (AWS). To access these data, you need to create a client to read data from Amazon's Simple Storage Service (s3) buckets, and you need to know some information about those buckets. The OpenUniverse2024 data preview contains simulations of the Roman Wide-Area Survey (WAS) and the Roman Time Domain Survey (TDS). In this tutorial, we will focus on the WAS. @@ -100,7 +100,7 @@ RUBIN_PREFIX = "openuniverse2024/rubin/preview" RUBIN_COADD_PATH = f"{RUBIN_PREFIX}/u/descdm/preview_data_step3_2877_19_w_2024_12/20240403T150003Z/deepCoadd_calexp/2877/19" ``` -## Roman Coadds +## 2. Roman Coadds The Nancy Grace Roman Space Telescope will carry out a wide-area survey (WAS) in the near infrared. The OpenUniverse2024 data preview includes coadded mosaics of simulated WAS data, created with the IMCOM algorithm (Rowe et al. 2011). Bands include F184, H158, J129, K213, Y106. In this section, we define some functions that make it convenient to retrieve a given cloud-hosted simulated Roman coadd based on position and filter. @@ -237,7 +237,7 @@ plt.imshow(coadd_roman['data'], origin='lower', plt.plot(*coord_arr_idx, 'r+', markersize=15) ``` -## Rubin Coadds +## 3. Rubin Coadds The OpenUniverse2024 data preview includes coadded mosaics in the following filters: u, g, r, i, z, y. In this section, we define some functions that make it convenient to retrieve a given cloud-hosted simulated Roman coadd based on position and filter. @@ -327,7 +327,7 @@ coadd_s3_fpath_rubin = get_rubin_coadd_fpath(filter_rubin) https_url(coadd_s3_fpath_rubin) ``` -## Compare simulated Roman and Rubin cutouts for a selected position +## 4. Compare simulated Roman and Rubin cutouts for a selected position +++ @@ -366,7 +366,7 @@ fig.suptitle(f"Cutouts at ({coord.ra}, {coord.dec}) with {cutout_size} size", fo plt.tight_layout(rect=[0, 0, 1, 0.97]) ``` -## Use Firefly to interactively identify a blended source +## 5. Use Firefly to interactively identify a blended source Clearly, the simulated Roman coadd has higher spatial resolution than the Rubin simulated coadd. Let's try to locate blended objects to compare in the simulated Rubin and Roman images. We will use Firefly's interactive visualization to make this task easier. @@ -452,7 +452,7 @@ point_region = f'icrs;point {coords_of_interest.ra.value}d {coords_of_interest.d fc.add_region_data(region_data=point_region, region_layer_id=roman_regions_id) ``` -## Plot cutouts of the identified blended source +## 6. Plot cutouts of the identified blended source ```{code-cell} ipython3 coadd_roman = get_roman_coadd(coords_of_interest, filter_roman) @@ -501,7 +501,7 @@ plt.tight_layout(rect=[0, 0, 1, 0.97]) # plt.savefig("plot.pdf", bbox_inches='tight', pad_inches=0.2) ``` -## Use Firefly to visualize the OpenUniverse2024 data preview catalogs +## 7. Use Firefly to visualize the OpenUniverse2024 data preview catalogs Let's inspect the properties of sources in the Rubin coadd image. For this we will use the input truth files present in S3 bucket. The OpenUniverse2024 data preview includes the input truth files that were used to create the simulated images. These files are in Parquet and HDF5 format, and include information about the properties of galaxies, stars, and transients. @@ -614,7 +614,7 @@ point_region = f'icrs;point {high_z_gal_coords.ra.value}d {high_z_gal_coords.dec fc.add_region_data(region_data=point_region, region_layer_id=roman_regions_id) ``` -## Plot 3-color Roman coadd containing your region of interest +## 8. Plot 3-color Roman coadd containing your region of interest Let's inspect WCS of Roman coadd first ```{code-cell} ipython3 diff --git a/tutorials/simulated-data/roman_hlss_number_density.md b/tutorials/simulated-data/roman_hlss_number_density.md index 56ff9a1a..81f201cf 100644 --- a/tutorials/simulated-data/roman_hlss_number_density.md +++ b/tutorials/simulated-data/roman_hlss_number_density.md @@ -452,7 +452,7 @@ else: print("There are no NaN values in the DataFrame.") ``` -## 2.0 Make some helpful histograms +## 2. Make some helpful histograms Do the numbers we are getting and their distributions make sense? - plot of the number of galaxies as a function of dec bin - plot the number of galaxies per square degree as a function of dec bin @@ -625,7 +625,7 @@ Figure Caption: Both plots have Declination on their X-axis. The top panel show +++ -## 3.0 Make Number density plot +## 3. Make Number density plot This is a first pass at making a number density plot with just a single data file. We divide the sample into 10 declination bins in order to do a jackknife sampling and obtain error bars for the plot. We choose to use constant dec bins instead of constant RA bins since area is function of cosine dec so we don't want to average over too much dec. ```{code-cell} ipython3 @@ -787,7 +787,7 @@ Figure Caption: Number density plot. Note that the jackknife error bars are dis del df ``` -## 4.0 Expand to out-of-memory sized catalogs +## 4. Expand to out-of-memory sized catalogs Originally this section attempted to use dask instead of pandas to hold the data and be able to do this work. Dask is supposed to be able to do this sort of magic in the background with out of memory catalogs, but in reality, this does not seem to be possible, or at least not possible with a week or so of working on it. Instead, we will read in the catalogs one at a time, store counts of mean and standard deviation per square degree and then move on to the next catalog. ```{code-cell} ipython3 diff --git a/tutorials/techniques-and-tools/Parallelize_Convolution.md b/tutorials/techniques-and-tools/Parallelize_Convolution.md index 04a4f0c6..59c5fa84 100644 --- a/tutorials/techniques-and-tools/Parallelize_Convolution.md +++ b/tutorials/techniques-and-tools/Parallelize_Convolution.md @@ -60,7 +60,7 @@ import scipy.signal import ray ``` -## Find the cpus available +## 1. Find the cpus available Find and print the number of cpus (taken from https://towardsdatascience.com/10x-faster-parallel-python-without-python-multiprocessing-e5017c93cce1) @@ -70,7 +70,7 @@ num_cpus = psutil.cpu_count(logical=True) print(num_cpus) ``` -## Process serially using a conventional loop +## 2. Process serially using a conventional loop +++ @@ -98,7 +98,7 @@ print("(scaled) conventional duration for {:d} iterations = {:.1f} seconds" .format(num_cpus*num_iter, duration_conv*num_cpus)) ``` -## Process in parallel using Ray +## 3. Process in parallel using Ray +++ @@ -137,7 +137,7 @@ print("Ray duration = {:.1f}, speedup = {:.2f}" ray.shutdown() ``` -## Process in parallel using multiprocessing +## 4. Process in parallel using multiprocessing +++ @@ -176,7 +176,7 @@ print("Multiprocessing duration = {:.1f}, speedup = {:.2f}" .format(duration_mp, duration_conv*num_cpus / duration_mp)) ``` -## Process using Dask +## 5. Process using Dask +++ @@ -212,7 +212,7 @@ print("Dask duration = {:.1f}, speedup = {:.2f}" client.close() ``` -## Conclusions +## 6. Conclusions +++ diff --git a/tutorials/techniques-and-tools/SEDs_in_Firefly.md b/tutorials/techniques-and-tools/SEDs_in_Firefly.md index 104aeb93..4c1661eb 100644 --- a/tutorials/techniques-and-tools/SEDs_in_Firefly.md +++ b/tutorials/techniques-and-tools/SEDs_in_Firefly.md @@ -80,7 +80,7 @@ import numpy as np import pyvo ``` -## Choose Target as Example +## 1. Choose Target as Example +++ @@ -90,7 +90,7 @@ From [Figure 10](https://doi.org/10.3847/1538-3881/ace32f#ajace32ff10) of the re target = SkyCoord(ra="05h27m36.37s", dec="+34d49m40.6s") ``` -## Get Photometry from IRSA +## 2. Get Photometry from IRSA +++ @@ -507,7 +507,7 @@ Note: After more columns are added in the conversion to spectral energy density, # Pan-STARRS, UKIDSS? ``` -## Convert Magnitudes to Energy Densities +## 3. Convert Magnitudes to Energy Densities +++ @@ -545,7 +545,7 @@ Run the following cell to view the full photometry table and its values for each phot_tbl ``` -## Upload to Firefly +## 4. Upload to Firefly +++ @@ -610,7 +610,7 @@ Display the SED plot in Firefly: fc.show_chart(layout=layout_s, data=trace_data) ``` -## Retrieve and Display Images +## 5. Retrieve and Display Images +++ diff --git a/tutorials/wise/NEOWISE_light_curve_demo.md b/tutorials/wise/NEOWISE_light_curve_demo.md index b60a84ff..87d66a54 100644 --- a/tutorials/wise/NEOWISE_light_curve_demo.md +++ b/tutorials/wise/NEOWISE_light_curve_demo.md @@ -52,7 +52,7 @@ import astropy.utils.data import urllib.parse ``` -## Instantiate the Firefly client +## 1. Instantiate the Firefly client There are two ways to initialize a Firefly client from Python, depending on whether you're running the notebook in JupyterLab or not. Assuming you have `jupyter-firefly-extensions` set up in your environment as explained [here](https://github.com/Caltech-IPAC/jupyter_firefly_extensions/blob/master/README.md), you can use `make_lab_client()` in JupyterLab, which will open the Firefly viewer in a new tab within the Lab. Otherwise, you can use `make_client()` in a Jupyter Notebook (or even a Python shell), which will open the Firefly viewer in a new web browser tab. @@ -68,7 +68,7 @@ fc = FireflyClient.make_client(url="https://irsa.ipac.caltech.edu/irsaviewer") fc.reinit_viewer() # to clean the state, if this cell ran earlier ``` -## Construct a TAP Query and display the retrieved table in Firefly +## 2. Construct a TAP Query and display the retrieved table in Firefly TAP search the 'Known Solar System Object Possible Association List' catalog from the NEOWISE-R database. The specific target we are looking for is minor planet `558 Carmen`. We can query this target using a TAP search through IRSA; the `table_url` is broken down as follows: @@ -106,7 +106,7 @@ Note that along with the table, firefly also displays the coverage and chart ass +++ -## Plot Light Curves in Firefly +## 3. Plot Light Curves in Firefly After retrieving the data and displaying it in the client, we can now create a light curve by plotting the Modified Julian Date ('mjd') in the abscissa and the magnitude from band W1 ('w1mpro') in the ordinate. We also flip the ordinate to accurately display magnitude. @@ -114,7 +114,7 @@ After retrieving the data and displaying it in the client, we can now create a l fc.show_xyplot(tbl_id='tableneo', xCol='mjd', yCol='w1mpro', yOptions='flip') ``` -## Overlay the catalog on a HiPS image +## 4. Overlay the catalog on a HiPS image Finally, we can overlay the catalog of data in the table onto a HiPS image of our choice, using the method [`show_hips`](https://caltech-ipac.github.io/firefly_client/api/firefly_client.FireflyClient.html#firefly_client.FireflyClient.show_hips). However, this method requires target coordinates for the object you want to analyze. @@ -127,7 +127,7 @@ fc.show_hips(viewer_id=viewer_id, plot_id='aHipsID1-1', hips_root_url = hips_url Title='HiPS-WISE', WorldPt=target) ``` -## Summary +## 5. Summary Firefly allows you to visualize data for specific targets. In conjuction with Astropy, one can manipulate a catalog of observations to display a light curve in an instantiated Firefly enviroment on their web browser. diff --git a/tutorials/wise/wise-allwise-catalog-demo.md b/tutorials/wise/wise-allwise-catalog-demo.md index 743fd1c5..045ea133 100644 --- a/tutorials/wise/wise-allwise-catalog-demo.md +++ b/tutorials/wise/wise-allwise-catalog-demo.md @@ -62,7 +62,7 @@ from matplotlib import pyplot as plt from pyarrow.fs import S3FileSystem ``` -## Setup catalog paths and query filters +## 1. Setup catalog paths and query filters +++ @@ -131,7 +131,7 @@ cone_pixels = hp.query_circle( print(f"cone_pixels contains {len(cone_pixels)} of a possible {hp.nside_to_npixel(nside)} pixels") ``` -## Example 1: Pandas with basic filters (magnitude limit and ra/dec polygon) +## 2. Pandas with basic filters (magnitude limit and ra/dec polygon) +++ @@ -171,7 +171,7 @@ pandas_df.describe() del pandas_df ``` -## Example 2: Pyarrow with advanced filters (color-color cuts for AGN) +## 3. Pyarrow with advanced filters (color-color cuts for AGN) +++ @@ -251,7 +251,7 @@ del pyarrow_ds del pyarrow_df ``` -## Example 3: Nearest-neighbor search (using pyarrow and astropy) +## 4. Nearest-neighbor search (using pyarrow and astropy) +++ @@ -326,7 +326,7 @@ del pyarrow_ds del neighbors_df ``` -## Schema Access +## 5. Schema Access +++ From ca45e9cde3b6b25dddab2cac916ed9b6d9480ba3 Mon Sep 17 00:00:00 2001 From: jkrick Date: Fri, 27 Feb 2026 09:16:26 -0800 Subject: [PATCH 2/3] Apply suggestions from code review Co-authored-by: Troy Raen --- tutorials/simulated-data/OpenUniverse2024Preview_Firefly.md | 2 +- tutorials/wise/wise-allwise-catalog-demo.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tutorials/simulated-data/OpenUniverse2024Preview_Firefly.md b/tutorials/simulated-data/OpenUniverse2024Preview_Firefly.md index a4058529..70e92510 100644 --- a/tutorials/simulated-data/OpenUniverse2024Preview_Firefly.md +++ b/tutorials/simulated-data/OpenUniverse2024Preview_Firefly.md @@ -86,7 +86,7 @@ from reproject import reproject_interp from io import BytesIO ``` -## 1. Learn where the OpenUniverse2024 data are hosted in the cloud. +## 1. Learn where the OpenUniverse2024 data are hosted in the cloud The OpenUniverse2024 data preview is hosted in the cloud via Amazon Web Services (AWS). To access these data, you need to create a client to read data from Amazon's Simple Storage Service (s3) buckets, and you need to know some information about those buckets. The OpenUniverse2024 data preview contains simulations of the Roman Wide-Area Survey (WAS) and the Roman Time Domain Survey (TDS). In this tutorial, we will focus on the WAS. diff --git a/tutorials/wise/wise-allwise-catalog-demo.md b/tutorials/wise/wise-allwise-catalog-demo.md index 045ea133..5025c5e5 100644 --- a/tutorials/wise/wise-allwise-catalog-demo.md +++ b/tutorials/wise/wise-allwise-catalog-demo.md @@ -135,7 +135,7 @@ print(f"cone_pixels contains {len(cone_pixels)} of a possible {hp.nside_to_npixe +++ -Load using [pandas.read_parquet](https://pandas.pydata.org/docs/reference/api/pandas.read_parquet.html). +Example 1: Load using [pandas.read_parquet](https://pandas.pydata.org/docs/reference/api/pandas.read_parquet.html). Filter for magnitudes above our w1mpro limit and a sky-area limited to the ra/dec polygon. Pandas actually uses either pyarrow or fastparquet to interact with parquet files. @@ -175,7 +175,7 @@ del pandas_df +++ -Load using [pyarrow.dataset.parquet_dataset](https://arrow.apache.org/docs/python/generated/pyarrow.dataset.parquet_dataset.html) and convert to pandas. +Example 2: Load using [pyarrow.dataset.parquet_dataset](https://arrow.apache.org/docs/python/generated/pyarrow.dataset.parquet_dataset.html) and convert to pandas. Useful for: 1. advanced filters that combine, compare, and/or create new columns. (this example) @@ -255,6 +255,7 @@ del pyarrow_df +++ +Example 3: Spatial search using Astropy. Nearest-neighbor searches and cone searches generally use the on-sky separation distance to determine the matches. It would be cumbersome to construct the new column and filter on it using the methods shown above because the separation distance is a fairly complicated function of ra and dec. However, we can get pretty fast results by filtering down to the HEALPix pixels that cover the region, loading all the data in those partitions, and then using astropy to compute the separations and find the matches. From fd339c79089c47b2f5245b42be50f2955ae47fd3 Mon Sep 17 00:00:00 2001 From: jkrick Date: Fri, 27 Feb 2026 09:18:32 -0800 Subject: [PATCH 3/3] Apply suggestions from code review --- tutorials/euclid/Euclid_ERO.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tutorials/euclid/Euclid_ERO.md b/tutorials/euclid/Euclid_ERO.md index 9f544eab..dc83fc06 100644 --- a/tutorials/euclid/Euclid_ERO.md +++ b/tutorials/euclid/Euclid_ERO.md @@ -97,7 +97,7 @@ import matplotlib as mpl Next, we define some parameters for `Matplotlib` plotting. ```{code-cell} ipython3 -## 1. Plotting stuff +# Plotting stuff mpl.rcParams['font.size'] = 14 mpl.rcParams['axes.labelpad'] = 7 mpl.rcParams['xtick.major.pad'] = 7 @@ -123,7 +123,7 @@ mpl.rcParams['hatch.linewidth'] = 1 def_cols = plt.rcParams['axes.prop_cycle'].by_key()['color'] ``` -## 2. Setting up the Environment +## 1. Setting up the Environment Next, we set up the environment. This includes * setting up an output data directory (will be created if it does not exist) @@ -148,7 +148,7 @@ cutout_size = 1.5 * u.arcmin # cutout size coord = SkyCoord.from_name('NGC 6397') ``` -## 3. Search Euclid ERO Images +## 2. Search Euclid ERO Images Now, we search for the Euclid ERO images using the `astroquery` package. Note that the Euclid ERO images are no in the cloud currently, but we access them directly from IRSA using IRSA's *Simple Image Access* (SIA) methods. @@ -227,7 +227,7 @@ Let's check out the summary table that we have created. We see that we have all summary_table ``` -## 4. Create Cutout Images +## 3. Create Cutout Images Now that we have a list of data products, we can create the cutouts. This is important as the full Euclid ERO images would be too large to run extraction and photometry software on them (they would simply fail due to memory issues). @@ -265,7 +265,7 @@ for ii,filt in tqdm(enumerate(filters)): hdu.header["FILTER"] = filt.upper() hdulcutout.append(hdu) -## 5. Save the HDUL cube: +# Save the HDUL cube: hdulcutout.writeto("./data/euclid_images_test.fits", overwrite=True) ``` @@ -294,7 +294,7 @@ for ii,filt in enumerate(filters): plt.show() ``` -## 6. Extract Sources and Measure their Photometry on the VIS Image +## 4. Extract Sources and Measure their Photometry on the VIS Image Now that we have the images in memory (and on disk - but we do not need them, yet), we can measure the fluxes of the individual stars. Our simple photometry pipeline has different parts: @@ -308,7 +308,7 @@ Our simple photometry pipeline has different parts: We start by extracting the sources using `sep`. We first isolate the data that we want to look at (the VIS image only). ```{code-cell} ipython3 -## 7. Get Data (this will be replaced later) +# Get Data (this will be replaced later) img = hdulcutout["VIS_SCIENCE"].data hdr = hdulcutout["VIS_SCIENCE"].header img[img == 0] = np.nan @@ -378,7 +378,7 @@ resimage = psfphot.make_residual_image(data = img-median, psf_shape = (9, 9)) We now want to add the best-fit coordinates (R.A. and Decl.) to the VIS photometry catalog. For this, we have to convert the image coordinates into sky coordinates using the WCS information. We will need these coordinates because we want to use them as positional priors for the photometry measurement on the NISP images. ```{code-cell} ipython3 -## 8. Add coordinates to catalog +# Add coordinates to catalog wcs1 = WCS(hdr) # VIS radec = wcs1.all_pix2world(phot["x_fit"],phot["y_fit"],0) phot["ra_fit"] = radec[0] @@ -424,7 +424,7 @@ ax1.set_yscale('log') plt.show() ``` -## 9. Measure the Photometry on the NISP Images +## 5. Measure the Photometry on the NISP Images We now have the photometry and the position of sources on the VIS image. We can now proceed with similar steps on the NISP images. Because the NISP PSF and pixel scale are larger that those of the VIS images, we utilize the advantage of position prior-based forced photometry. For this, we use the positions of the VIS measurements and perform PSF fitting on the NISP image using these priors. @@ -508,7 +508,7 @@ ax2.plot(phot2["x_fit"], phot2["y_fit"] , "o", markersize=8 , markeredgecolor="r plt.show() ``` -## 10. Load Gaia Catalog +## 6. Load Gaia Catalog We now load the Gaia sources at the location of the globular clusters. The goal is to compare the photometry of Gaia to the one derived above for the Euclid VIS and NISP images. This is scientifically useful, for example we can compute the colors of the stars in the Gaia optical bands and the Euclid near-IR bands. To search for Gaia sources, we use `astroquery` again. @@ -563,7 +563,7 @@ ax2.set_title("NISP") plt.show() ``` -## 11. Match the Gaia Catalog to the VIS and NISP Catalogs +## 7. Match the Gaia Catalog to the VIS and NISP Catalogs Now, we match the Gaia source positions to the extracted sources in the VIS and NISP images. @@ -628,7 +628,7 @@ ax1.set_ylabel("I$_E$ [mag]") plt.show() ``` -## 12. Visualization with Firefly +## 8. Visualization with Firefly At the end of this Notebook, we demonstrate how we can visualize the images and catalogs created above in `Firefly`.