diff --git a/lib/FredyPipelineExecutioner.js b/lib/FredyPipelineExecutioner.js index 58221060..70fb15af 100755 --- a/lib/FredyPipelineExecutioner.js +++ b/lib/FredyPipelineExecutioner.js @@ -19,6 +19,7 @@ import { distanceMeters } from './services/listings/distanceCalculator.js'; import { getUserSettings } from './services/storage/settingsStorage.js'; import { updateListingDistance } from './services/storage/listingsStorage.js'; import booleanPointInPolygon from '@turf/boolean-point-in-polygon'; +import { extractNumber } from './utils/extract-number.js'; /** * @typedef {Object} Listing @@ -26,8 +27,13 @@ import booleanPointInPolygon from '@turf/boolean-point-in-polygon'; * @property {string} title Title or headline of the listing. * @property {string} [address] Optional address/location text. * @property {string} [price] Optional price text/value. + * @property {string} [size] Optional size text/value. + * @property {string} [rooms] Optional number of rooms text/value. * @property {string} [url] Link to the listing detail page. * @property {any} [meta] Provider-specific additional metadata. + * @property {number | null} [roomsInt] Optional number of rooms. + * @property {number | null} [sizeInt] Optional size of the listing. + * @property {number | null} [priceInt] Optional price of the listing. */ /** @@ -48,7 +54,9 @@ import booleanPointInPolygon from '@turf/boolean-point-in-polygon'; * 5) Identify new listings (vs. previously stored hashes) * 6) Persist new listings * 7) Filter out entries similar to already seen ones - * 8) Dispatch notifications + * 8) Filter out entries that do not match the job's specFilter + * 9) Filter out entries that do not match the job's spatialFilter + * 10) Dispatch notifications */ class FredyPipelineExecutioner { /** @@ -62,20 +70,25 @@ class FredyPipelineExecutioner { * @param {string} providerConfig.crawlContainer CSS selector for the container holding listing items. * @param {(raw:any)=>Listing} providerConfig.normalize Function to convert raw scraped data into a Listing shape. * @param {(listing:Listing)=>boolean} providerConfig.filter Function to filter out unwanted listings. + * + * @param {Object} job Job configuration. + * @param {string} job.id Job ID. + * @param {Object} job.notificationAdapter Notification configuration passed to notification adapters. + * @param {Object | null} job.spatialFilter Optional spatial filter configuration. + * @param {Object | null} job.specFilter Optional listing specifications (minRooms, minSize, maxPrice). + * * @param {(url:string, waitForSelector?:string)=>Promise|Promise} [providerConfig.getListings] Optional override to fetch listings. - * @param {Object} notificationConfig Notification configuration passed to notification adapters. - * @param {Object} spatialFilter Optional spatial filter configuration. * @param {string} providerId The ID of the provider currently in use. - * @param {string} jobKey Key of the job that is currently running (from within the config). * @param {SimilarityCache} similarityCache Cache instance for checking similar entries. * @param browser */ - constructor(providerConfig, notificationConfig, spatialFilter, providerId, jobKey, similarityCache, browser) { + constructor(providerConfig, job, providerId, similarityCache, browser) { this._providerConfig = providerConfig; - this._notificationConfig = notificationConfig; - this._spatialFilter = spatialFilter; + this._jobNotificationConfig = job.notificationAdapter; + this._jobKey = job.id; + this._jobSpecFilter = job.specFilter; + this._jobSpatialFilter = job.spatialFilter; this._providerId = providerId; - this._jobKey = jobKey; this._similarityCache = similarityCache; this._browser = browser; } @@ -96,6 +109,7 @@ class FredyPipelineExecutioner { .then(this._save.bind(this)) .then(this._calculateDistance.bind(this)) .then(this._filterBySimilarListings.bind(this)) + .then(this._filterBySpecs.bind(this)) .then(this._filterByArea.bind(this)) .then(this._notify.bind(this)) .catch(this._handleError.bind(this)); @@ -128,16 +142,15 @@ class FredyPipelineExecutioner { * @returns {Promise} Resolves with listings that are within the area (or not filtered if no area is set). */ _filterByArea(newListings) { - const polygonFeatures = this._spatialFilter?.features?.filter((f) => f.geometry?.type === 'Polygon'); + const polygonFeatures = this._jobSpatialFilter?.features?.filter((f) => f.geometry?.type === 'Polygon'); // If no area filter is set, return all listings if (!polygonFeatures?.length) { return newListings; } - const filteredIds = []; // Filter listings by area - keep only those within the polygon - const keptListings = newListings.filter((listing) => { + const filteredListings = newListings.filter((listing) => { // If listing doesn't have coordinates, keep it (don't filter out) if (listing.latitude == null || listing.longitude == null) { return true; @@ -147,18 +160,34 @@ class FredyPipelineExecutioner { const point = [listing.longitude, listing.latitude]; // GeoJSON format: [lon, lat] const isInPolygon = polygonFeatures.some((feature) => booleanPointInPolygon(point, feature)); - if (!isInPolygon) { - filteredIds.push(listing.id); - } - return isInPolygon; }); - if (filteredIds.length > 0) { - deleteListingsById(filteredIds); + return filteredListings; + } + + /** + * Filter listings based on its specifications (minRooms, minSize, maxPrice). + * + * @param {Listing[]} newListings New listings to filter. + * @returns {Promise} Resolves with listings that pass the specification filters. + */ + _filterBySpecs(newListings) { + const { minRooms, minSize, maxPrice } = this._jobSpecFilter || {}; + + // If no specs are set, return all listings + if (!minRooms && !minSize && !maxPrice) { + return newListings; } - return keptListings; + const filtered = newListings.filter((listing) => { + if (minRooms && listing.roomsInt && listing.roomsInt < minRooms) return false; + if (minSize && listing.sizeInt && listing.sizeInt < minSize) return false; + if (maxPrice && listing.priceInt && listing.priceInt > maxPrice) return false; + return true; + }); + + return filtered; } /** @@ -195,7 +224,16 @@ class FredyPipelineExecutioner { * @returns {Listing[]} Normalized listings. */ _normalize(listings) { - return listings.map(this._providerConfig.normalize); + return listings.map((listing) => { + const normalized = this._providerConfig.normalize(listing); + // TODO: every provider should return price, size and rooms in type number. This makes it more strong and strict of the provider output. String formats like "m², Zi,..." should not be part and can be added on fe or massages. Move this logic into the provider-specific normalize function. + return { + ...normalized, + priceInt: extractNumber(normalized.price), + sizeInt: extractNumber(normalized.size), + roomsInt: extractNumber(normalized.rooms), + }; + }); } /** @@ -206,7 +244,13 @@ class FredyPipelineExecutioner { * @returns {Listing[]} Filtered listings that pass validation and provider filter. */ _filter(listings) { - const keys = Object.keys(this._providerConfig.crawlFields); + // i removed it because crawlFields might be different than fields which are required. + // like for kleinanzeigen we have tags (includes multiple fields) but will be than extract at normalize, and deleted because its only internal used. + // I would suggest that we define a standard list like (id, price, rooms, size, title, link, description, address, image, url) + // it might be that some of this props value is null, wich is ok without id, link, title + // Also this might be not needed when using typings with typescript. I would suggest to move the whole project to typescript to have save typings. + //const keys = Object.keys(this._providerConfig.crawlFields); + const keys = ['id', 'link', 'title']; const filteredListings = listings.filter((item) => keys.every((key) => key in item)); return filteredListings.filter(this._providerConfig.filter); } @@ -240,7 +284,7 @@ class FredyPipelineExecutioner { if (newListings.length === 0) { throw new NoNewListingsWarning(); } - const sendNotifications = notify.send(this._providerId, newListings, this._notificationConfig, this._jobKey); + const sendNotifications = notify.send(this._providerId, newListings, this._jobNotificationConfig, this._jobKey); return Promise.all(sendNotifications).then(() => newListings); } diff --git a/lib/api/routes/jobRouter.js b/lib/api/routes/jobRouter.js index 1798cf77..f059bc80 100644 --- a/lib/api/routes/jobRouter.js +++ b/lib/api/routes/jobRouter.js @@ -172,6 +172,7 @@ jobRouter.post('/', async (req, res) => { enabled, shareWithUsers = [], spatialFilter = null, + specFilter = null, } = req.body; const settings = await getSettings(); try { @@ -197,6 +198,7 @@ jobRouter.post('/', async (req, res) => { notificationAdapter, shareWithUsers, spatialFilter, + specFilter, }); } catch (error) { res.send(new Error(error)); diff --git a/lib/api/routes/listingsRouter.js b/lib/api/routes/listingsRouter.js index cf953333..179c7938 100644 --- a/lib/api/routes/listingsRouter.js +++ b/lib/api/routes/listingsRouter.js @@ -27,6 +27,7 @@ listingsRouter.get('/table', async (req, res) => { sortfield = null, sortdir = 'asc', freeTextFilter, + filterByJobSettings, } = req.query || {}; // normalize booleans (accept true, 'true', 1, '1' for true; false, 'false', 0, '0' for false) @@ -37,6 +38,7 @@ listingsRouter.get('/table', async (req, res) => { }; const normalizedActivity = toBool(activityFilter); const normalizedWatch = toBool(watchListFilter); + const normalizedFilterByJobSettings = toBool(filterByJobSettings) ?? true; let jobFilter = null; let jobIdFilter = null; @@ -56,6 +58,7 @@ listingsRouter.get('/table', async (req, res) => { jobIdFilter: jobIdFilter, providerFilter, watchListFilter: normalizedWatch, + filterByJobSettings: normalizedFilterByJobSettings, sortField: sortfield || null, sortDir: sortdir === 'desc' ? 'desc' : 'asc', userId: req.session.currentUser, diff --git a/lib/provider/kleinanzeigen.js b/lib/provider/kleinanzeigen.js index d72c474c..09a25a91 100755 --- a/lib/provider/kleinanzeigen.js +++ b/lib/provider/kleinanzeigen.js @@ -10,10 +10,14 @@ let appliedBlackList = []; let appliedBlacklistedDistricts = []; function normalize(o) { - const size = o.size || '--- m²'; + const parts = (o.tags || '').split('·').map((p) => p.trim()); + const size = parts.find((p) => p.includes('m²')) || '--- m²'; + const rooms = parts.find((p) => p.includes('Zi.')) || '--- Zi.'; const id = buildHash(o.id, o.price); const link = `https://www.kleinanzeigen.de${o.link}`; - return Object.assign(o, { id, size, link }); + + delete o.tags; + return Object.assign(o, { id, size, rooms, link }); } function applyBlacklist(o) { @@ -33,7 +37,7 @@ const config = { crawlFields: { id: '.aditem@data-adid | int', price: '.aditem-main--middle--price-shipping--price | removeNewline | trim', - size: '.aditem-main .text-module-end | removeNewline | trim', + tags: '.aditem-main--middle--tags | removeNewline | trim', title: '.aditem-main .text-module-begin a | removeNewline | trim', link: '.aditem-main .text-module-begin a@href | removeNewline | trim', description: '.aditem-main .aditem-main--middle--description | removeNewline | trim', diff --git a/lib/services/jobs/jobExecutionService.js b/lib/services/jobs/jobExecutionService.js index b8324b88..99408f79 100644 --- a/lib/services/jobs/jobExecutionService.js +++ b/lib/services/jobs/jobExecutionService.js @@ -178,15 +178,7 @@ export function initJobExecutionService({ providers, settings, intervalMs }) { browser = await puppeteerExtractor.launchBrowser(matchedProvider.config.url, {}); } - await new FredyPipelineExecutioner( - matchedProvider.config, - job.notificationAdapter, - job.spatialFilter, - prov.id, - job.id, - similarityCache, - browser, - ).execute(); + await new FredyPipelineExecutioner(matchedProvider.config, job, prov.id, similarityCache, browser).execute(); } catch (err) { logger.error(err); } diff --git a/lib/services/storage/jobStorage.js b/lib/services/storage/jobStorage.js index bf88003f..a734c5c2 100644 --- a/lib/services/storage/jobStorage.js +++ b/lib/services/storage/jobStorage.js @@ -31,6 +31,7 @@ export const upsertJob = ({ userId, shareWithUsers = [], spatialFilter = null, + specFilter = null, }) => { const id = jobId || nanoid(); const existing = SqliteConnection.query(`SELECT id, user_id FROM jobs WHERE id = @id LIMIT 1`, { id })[0]; @@ -44,7 +45,8 @@ export const upsertJob = ({ provider = @provider, notification_adapter = @notification_adapter, shared_with_user = @shareWithUsers, - spatial_filter = @spatialFilter + spatial_filter = @spatialFilter, + spec_filter = @specFilter WHERE id = @id`, { id, @@ -55,12 +57,13 @@ export const upsertJob = ({ provider: toJson(provider ?? []), notification_adapter: toJson(notificationAdapter ?? []), spatialFilter: spatialFilter ? toJson(spatialFilter) : null, + specFilter: specFilter ? toJson(specFilter) : null, }, ); } else { SqliteConnection.execute( - `INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter, shared_with_user, spatial_filter) - VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter, @shareWithUsers, @spatialFilter)`, + `INSERT INTO jobs (id, user_id, enabled, name, blacklist, provider, notification_adapter, shared_with_user, spatial_filter, spec_filter) + VALUES (@id, @user_id, @enabled, @name, @blacklist, @provider, @notification_adapter, @shareWithUsers, @spatialFilter, @specFilter)`, { id, user_id: ownerId, @@ -71,6 +74,7 @@ export const upsertJob = ({ shareWithUsers: toJson(shareWithUsers ?? []), notification_adapter: toJson(notificationAdapter ?? []), spatialFilter: spatialFilter ? toJson(spatialFilter) : null, + specFilter: specFilter ? toJson(specFilter) : null, }, ); } @@ -92,6 +96,7 @@ export const getJob = (jobId) => { j.shared_with_user, j.notification_adapter AS notificationAdapter, j.spatial_filter AS spatialFilter, + j.spec_filter AS specFilter, (SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings FROM jobs j WHERE j.id = @id @@ -107,6 +112,7 @@ export const getJob = (jobId) => { shared_with_user: fromJson(row.shared_with_user, []), notificationAdapter: fromJson(row.notificationAdapter, []), spatialFilter: fromJson(row.spatialFilter, null), + specFilter: fromJson(row.specFilter, null), }; }; @@ -157,6 +163,7 @@ export const getJobs = () => { j.shared_with_user, j.notification_adapter AS notificationAdapter, j.spatial_filter AS spatialFilter, + j.spec_filter AS specFilter, (SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings FROM jobs j WHERE j.enabled = 1 @@ -170,6 +177,7 @@ export const getJobs = () => { shared_with_user: fromJson(row.shared_with_user, []), notificationAdapter: fromJson(row.notificationAdapter, []), spatialFilter: fromJson(row.spatialFilter, null), + specFilter: fromJson(row.specFilter, null), })); }; @@ -260,6 +268,7 @@ export const queryJobs = ({ j.shared_with_user, j.notification_adapter AS notificationAdapter, j.spatial_filter AS spatialFilter, + j.spec_filter AS specFilter, (SELECT COUNT(1) FROM listings l WHERE l.job_id = j.id AND l.is_active = 1 AND l.manually_deleted = 0) AS numberOfFoundListings FROM jobs j ${whereSql} @@ -276,6 +285,7 @@ export const queryJobs = ({ shared_with_user: fromJson(row.shared_with_user, []), notificationAdapter: fromJson(row.notificationAdapter, []), spatialFilter: fromJson(row.spatialFilter, null), + specFilter: fromJson(row.specFilter, null), })); return { totalNumber, page: safePage, result }; diff --git a/lib/services/storage/listingsStorage.js b/lib/services/storage/listingsStorage.js index d8a54d02..31f9395b 100755 --- a/lib/services/storage/listingsStorage.js +++ b/lib/services/storage/listingsStorage.js @@ -174,9 +174,9 @@ export const storeListings = (jobId, providerId, listings) => { SqliteConnection.withTransaction((db) => { const stmt = db.prepare( - `INSERT INTO listings (id, hash, provider, job_id, price, size, title, image_url, description, address, + `INSERT INTO listings (id, hash, provider, job_id, price, size, rooms, title, image_url, description, address, link, created_at, is_active, latitude, longitude) - VALUES (@id, @hash, @provider, @job_id, @price, @size, @title, @image_url, @description, @address, @link, + VALUES (@id, @hash, @provider, @job_id, @price, @size, @rooms, @title, @image_url, @description, @address, @link, @created_at, 1, @latitude, @longitude) ON CONFLICT(job_id, hash) DO NOTHING`, ); @@ -187,8 +187,9 @@ export const storeListings = (jobId, providerId, listings) => { hash: item.id, provider: providerId, job_id: jobId, - price: extractNumber(item.price), - size: extractNumber(item.size), + price: item.priceInt, + size: item.sizeInt, + rooms: item.roomsInt, title: item.title, image_url: item.image, description: item.description, @@ -202,19 +203,6 @@ export const storeListings = (jobId, providerId, listings) => { } }); - /** - * Extract the first number from a string like "1.234 €" or "70 m²". - * Removes dots/commas before parsing. Returns null on invalid input. - * @param {string|undefined|null} str - * @returns {number|null} - */ - function extractNumber(str) { - if (!str) return null; - const cleaned = str.replace(/\./g, '').replace(',', '.'); - const num = parseFloat(cleaned); - return isNaN(num) ? null : num; - } - /** * Remove any parentheses segments (including surrounding whitespace) from a string. * Returns null for empty input. @@ -255,6 +243,7 @@ export const queryListings = ({ providerFilter, watchListFilter, freeTextFilter, + filterByJobSettings, sortField = null, sortDir = 'asc', userId = null, @@ -308,6 +297,15 @@ export const queryListings = ({ whereParts.push('(wl.id IS NULL)'); } + // filterByJobSettings: when true, filter listings by spec_filter in job settings + if (filterByJobSettings === true) { + whereParts.push(`( + (json_extract(j.spec_filter, '$.minRooms') IS NULL OR l.rooms IS NULL OR l.rooms >= json_extract(j.spec_filter, '$.minRooms')) AND + (json_extract(j.spec_filter, '$.minSize') IS NULL OR l.size IS NULL OR l.size >= json_extract(j.spec_filter, '$.minSize')) AND + (json_extract(j.spec_filter, '$.maxPrice') IS NULL OR l.price IS NULL OR l.price <= json_extract(j.spec_filter, '$.maxPrice')) + )`); + } + // Build whereSql (filtering by manually_deleted = 0) whereParts.push('(l.manually_deleted = 0)'); diff --git a/lib/services/storage/migrations/sql/12.add-listing-specs.js b/lib/services/storage/migrations/sql/12.add-listing-specs.js new file mode 100644 index 00000000..c7b7ec79 --- /dev/null +++ b/lib/services/storage/migrations/sql/12.add-listing-specs.js @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2026 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +export function up(db) { + db.exec(` + ALTER TABLE jobs ADD COLUMN spec_filter JSONB DEFAULT NULL; + `); +} diff --git a/lib/services/storage/migrations/sql/13.add-rooms-to-listings.js b/lib/services/storage/migrations/sql/13.add-rooms-to-listings.js new file mode 100644 index 00000000..870a8b74 --- /dev/null +++ b/lib/services/storage/migrations/sql/13.add-rooms-to-listings.js @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2026 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +export function up(db) { + db.exec(` + ALTER TABLE listings ADD COLUMN rooms INTEGER; + `); +} diff --git a/lib/utils/extract-number.js b/lib/utils/extract-number.js new file mode 100644 index 00000000..ec74545a --- /dev/null +++ b/lib/utils/extract-number.js @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2026 by Christian Kellner. + * Licensed under Apache-2.0 with Commons Clause and Attribution/Naming Clause + */ + +/** + * Extract the first number from a string like "1.234 €" or "70 m²". + * Removes dots/commas before parsing. Returns null on invalid input. + * @param {string|undefined|null} str + * @returns {number|null} + */ +export const extractNumber = (str) => { + if (!str) return null; + const cleaned = str.replace(/\./g, '').replace(',', '.'); + const num = parseFloat(cleaned); + return isNaN(num) ? null : num; +}; diff --git a/test/provider/einsAImmobilien.test.js b/test/provider/einsAImmobilien.test.js index e0ebb02b..642231da 100644 --- a/test/provider/einsAImmobilien.test.js +++ b/test/provider/einsAImmobilien.test.js @@ -13,15 +13,16 @@ describe('#einsAImmobilien testsuite()', () => { provider.init(providerConfig.einsAImmobilien, [], []); it('should test einsAImmobilien provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: '', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; + return await new Promise((resolve) => { - const fredy = new Fredy( - provider.config, - null, - null, - provider.metaInformation.id, - 'einsAImmobilien', - similarityCache, - ); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + fredy.execute().then((listings) => { expect(listings).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/immobilienDe.test.js b/test/provider/immobilienDe.test.js index a17485c4..655c9a43 100644 --- a/test/provider/immobilienDe.test.js +++ b/test/provider/immobilienDe.test.js @@ -13,8 +13,16 @@ describe('#immobilien.de testsuite()', () => { provider.init(providerConfig.immobilienDe, [], []); it('should test immobilien.de provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'test1', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; + return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'test1', similarityCache); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + fredy.execute().then((listing) => { expect(listing).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/immoscout.test.js b/test/provider/immoscout.test.js index 29b12e8c..3720323b 100644 --- a/test/provider/immoscout.test.js +++ b/test/provider/immoscout.test.js @@ -13,8 +13,15 @@ describe('#immoscout provider testsuite()', () => { provider.init(providerConfig.immoscout, [], []); it('should test immoscout provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: '', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; + return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, '', similarityCache); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); fredy.execute().then((listings) => { expect(listings).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/immoswp.test.js b/test/provider/immoswp.test.js index 8f34d3b1..2545f8c3 100644 --- a/test/provider/immoswp.test.js +++ b/test/provider/immoswp.test.js @@ -13,8 +13,16 @@ describe('#immoswp testsuite()', () => { provider.init(providerConfig.immoswp, [], []); it('should test immoswp provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'immoswp', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; + return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'immoswp', similarityCache); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + fredy.execute().then((listing) => { expect(listing).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/immowelt.test.js b/test/provider/immowelt.test.js index 9e5a0349..ebc6d07c 100644 --- a/test/provider/immowelt.test.js +++ b/test/provider/immowelt.test.js @@ -12,9 +12,16 @@ import * as provider from '../../lib/provider/immowelt.js'; describe('#immowelt testsuite()', () => { it('should test immowelt provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'immowelt', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; provider.init(providerConfig.immowelt, [], []); - const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'immowelt', similarityCache); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + const listing = await fredy.execute(); expect(listing).to.be.a('array'); diff --git a/test/provider/kleinanzeigen.test.js b/test/provider/kleinanzeigen.test.js index 628cadff..42ab4928 100644 --- a/test/provider/kleinanzeigen.test.js +++ b/test/provider/kleinanzeigen.test.js @@ -12,16 +12,16 @@ import * as provider from '../../lib/provider/kleinanzeigen.js'; describe('#kleinanzeigen testsuite()', () => { it('should test kleinanzeigen provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'kleinanzeigen', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; provider.init(providerConfig.kleinanzeigen, [], []); return await new Promise((resolve) => { - const fredy = new Fredy( - provider.config, - null, - null, - provider.metaInformation.id, - 'kleinanzeigen', - similarityCache, - ); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + fredy.execute().then((listing) => { expect(listing).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/mcMakler.test.js b/test/provider/mcMakler.test.js index d35414f9..e556b70b 100644 --- a/test/provider/mcMakler.test.js +++ b/test/provider/mcMakler.test.js @@ -12,9 +12,16 @@ import * as provider from '../../lib/provider/mcMakler.js'; describe('#mcMakler testsuite()', () => { it('should test mcMakler provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'mcMakler', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; provider.init(providerConfig.mcMakler, []); - const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'mcMakler', similarityCache); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + const listing = await fredy.execute(); expect(listing).to.be.a('array'); diff --git a/test/provider/neubauKompass.test.js b/test/provider/neubauKompass.test.js index 1182bb29..1606bd52 100644 --- a/test/provider/neubauKompass.test.js +++ b/test/provider/neubauKompass.test.js @@ -13,15 +13,16 @@ describe('#neubauKompass testsuite()', () => { provider.init(providerConfig.neubauKompass, [], []); it('should test neubauKompass provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'neubauKompass', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; + return await new Promise((resolve) => { - const fredy = new Fredy( - provider.config, - null, - null, - provider.metaInformation.id, - 'neubauKompass', - similarityCache, - ); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + fredy.execute().then((listing) => { expect(listing).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/ohneMakler.test.js b/test/provider/ohneMakler.test.js index eec70f02..1d5d9200 100644 --- a/test/provider/ohneMakler.test.js +++ b/test/provider/ohneMakler.test.js @@ -12,9 +12,16 @@ import * as provider from '../../lib/provider/ohneMakler.js'; describe('#ohneMakler testsuite()', () => { it('should test ohneMakler provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'ohneMakler', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; provider.init(providerConfig.ohneMakler, []); - const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'ohneMakler', similarityCache); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + const listing = await fredy.execute(); expect(listing).to.be.a('array'); diff --git a/test/provider/regionalimmobilien24.test.js b/test/provider/regionalimmobilien24.test.js index 2a4dfd97..cb7ed38b 100644 --- a/test/provider/regionalimmobilien24.test.js +++ b/test/provider/regionalimmobilien24.test.js @@ -12,16 +12,16 @@ import * as provider from '../../lib/provider/regionalimmobilien24.js'; describe('#regionalimmobilien24 testsuite()', () => { it('should test regionalimmobilien24 provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'regionalimmobilien24', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; provider.init(providerConfig.regionalimmobilien24, []); - const fredy = new Fredy( - provider.config, - null, - null, - provider.metaInformation.id, - 'regionalimmobilien24', - similarityCache, - ); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + const listing = await fredy.execute(); expect(listing).to.be.a('array'); diff --git a/test/provider/sparkasse.test.js b/test/provider/sparkasse.test.js index 875f8af4..cd01461a 100644 --- a/test/provider/sparkasse.test.js +++ b/test/provider/sparkasse.test.js @@ -12,9 +12,16 @@ import * as provider from '../../lib/provider/sparkasse.js'; describe('#sparkasse testsuite()', () => { it('should test sparkasse provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'sparkasse', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; provider.init(providerConfig.sparkasse, []); - const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'sparkasse', similarityCache); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + const listing = await fredy.execute(); expect(listing).to.be.a('array'); diff --git a/test/provider/wgGesucht.test.js b/test/provider/wgGesucht.test.js index ad1ca9c0..d051c480 100644 --- a/test/provider/wgGesucht.test.js +++ b/test/provider/wgGesucht.test.js @@ -13,8 +13,16 @@ describe('#wgGesucht testsuite()', () => { provider.init(providerConfig.wgGesucht, [], []); it('should test wgGesucht provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'wgGesucht', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; + return await new Promise((resolve) => { - const fredy = new Fredy(provider.config, null, null, provider.metaInformation.id, 'wgGesucht', similarityCache); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + fredy.execute().then((listing) => { expect(listing).to.be.a('array'); const notificationObj = get(); diff --git a/test/provider/wohnungsboerse.test.js b/test/provider/wohnungsboerse.test.js index 2f270720..9510e5ac 100644 --- a/test/provider/wohnungsboerse.test.js +++ b/test/provider/wohnungsboerse.test.js @@ -13,15 +13,16 @@ describe('#wohnungsboerse testsuite()', () => { provider.init(providerConfig.wohnungsboerse, [], []); it('should test wohnungsboerse provider', async () => { const Fredy = await mockFredy(); + const mockedJob = { + id: 'wohnungsboerse', + notificationAdapter: null, + spatialFilter: null, + specFilter: null, + }; + return await new Promise((resolve) => { - const fredy = new Fredy( - provider.config, - null, - null, - provider.metaInformation.id, - 'wohnungsboerse', - similarityCache, - ); + const fredy = new Fredy(provider.config, mockedJob, provider.metaInformation.id, similarityCache); + fredy.execute().then((listings) => { expect(listings).to.be.a('array'); const notificationObj = get(); diff --git a/ui/src/components/grid/listings/ListingsGrid.jsx b/ui/src/components/grid/listings/ListingsGrid.jsx index d555db21..272d76d5 100644 --- a/ui/src/components/grid/listings/ListingsGrid.jsx +++ b/ui/src/components/grid/listings/ListingsGrid.jsx @@ -19,6 +19,7 @@ import { Select, Popover, Empty, + Switch, } from '@douyinfe/semi-ui-19'; import { IconBriefcase, @@ -33,6 +34,8 @@ import { IconFilter, IconActivity, IconEyeOpened, + IconGridView, + IconExpand, } from '@douyinfe/semi-icons'; import { useNavigate } from 'react-router-dom'; import ListingDeletionModal from '../../ListingDeletionModal.jsx'; @@ -64,6 +67,7 @@ const ListingsGrid = () => { const [jobNameFilter, setJobNameFilter] = useState(null); const [activityFilter, setActivityFilter] = useState(null); const [providerFilter, setProviderFilter] = useState(null); + const [filterByJobSettings, setFilterByJobSettings] = useState(true); const [showFilterBar, setShowFilterBar] = useState(false); const [deleteModalVisible, setDeleteModalVisible] = useState(false); @@ -76,13 +80,23 @@ const ListingsGrid = () => { sortfield: sortField, sortdir: sortDir, freeTextFilter, - filter: { watchListFilter, jobNameFilter, activityFilter, providerFilter }, + filter: { watchListFilter, jobNameFilter, activityFilter, providerFilter, filterByJobSettings }, }); }; useEffect(() => { loadData(); - }, [page, sortField, sortDir, freeTextFilter, providerFilter, activityFilter, jobNameFilter, watchListFilter]); + }, [ + page, + sortField, + sortDir, + freeTextFilter, + providerFilter, + activityFilter, + jobNameFilter, + watchListFilter, + filterByJobSettings, + ]); const handleFilterChange = useMemo(() => debounce((value) => setFreeTextFilter(value), 500), []); @@ -195,6 +209,13 @@ const ListingsGrid = () => { ))} + +
+ setFilterByJobSettings(val)} size="small" /> + + Job Filters + +
@@ -281,9 +302,21 @@ const ListingsGrid = () => { {cap(item.title)} - } size="small"> - {item.price} € - + + } size="small"> + {item.price} € + + {item.size && ( + } size="small"> + {item.size} m² + + )} + {item.rooms && ( + } size="small"> + {item.rooms} Rooms + + )} + } @@ -293,12 +326,14 @@ const ListingsGrid = () => { > {item.address || 'No address provided'} - }> - {timeService.format(item.created_at, false)} - - }> - {item.provider.charAt(0).toUpperCase() + item.provider.slice(1)} - + + }> + {item.provider.charAt(0).toUpperCase() + item.provider.slice(1)} + + }> + {timeService.format(item.created_at, false)} + + {item.distance_to_destination ? ( }> {item.distance_to_destination} m to chosen address diff --git a/ui/src/components/grid/listings/ListingsGrid.less b/ui/src/components/grid/listings/ListingsGrid.less index d9a2473a..9bf6212a 100644 --- a/ui/src/components/grid/listings/ListingsGrid.less +++ b/ui/src/components/grid/listings/ListingsGrid.less @@ -45,6 +45,7 @@ } &--inactive { + .listingsGrid__imageContainer, .listingsGrid__content { opacity: 0.6; @@ -135,4 +136,16 @@ background: var(--semi-color-primary-hover); } } + + // Ensure icons and text are vertically aligned + .semi-typography { + display: inline-flex; + align-items: center; + + .semi-typography-icon { + display: flex; + align-items: center; + margin-top: 1px; // Minor nudge if needed, but flex should handle most + } + } } diff --git a/ui/src/views/jobs/mutation/JobMutation.jsx b/ui/src/views/jobs/mutation/JobMutation.jsx index c00d3f6f..623f0f33 100644 --- a/ui/src/views/jobs/mutation/JobMutation.jsx +++ b/ui/src/views/jobs/mutation/JobMutation.jsx @@ -24,9 +24,15 @@ import { IconPlayCircle, IconPlusCircle, IconUser, - IconClear, + IconFilter, } from '@douyinfe/semi-icons'; +const SPEC_FILTERS = [ + { key: 'maxPrice', translation: 'Max Price' }, + { key: 'minSize', translation: 'Min Size (m²)' }, + { key: 'minRooms', translation: 'Min Rooms' }, +]; + export default function JobMutator() { const jobs = useSelector((state) => state.jobsData.jobs); const shareableUserList = useSelector((state) => state.jobsData.shareableUserList); @@ -46,6 +52,7 @@ export default function JobMutator() { const defaultEnabled = sourceJob?.enabled ?? true; const defaultShareWithUsers = sourceJob?.shared_with_user ?? []; const defaultSpatialFilter = sourceJob?.spatialFilter || null; + const defaultSpecFilter = sourceJob?.specFilter || null; const [providerToEdit, setProviderToEdit] = useState(null); const [providerCreationVisible, setProviderCreationVisibility] = useState(false); @@ -58,6 +65,7 @@ export default function JobMutator() { const [shareWithUsers, setShareWithUsers] = useState(defaultShareWithUsers); const [enabled, setEnabled] = useState(defaultEnabled); const [spatialFilter, setSpatialFilter] = useState(defaultSpatialFilter); + const [specFilter, setSpecFilter] = useState(defaultSpecFilter); const navigate = useNavigate(); const actions = useActions(); @@ -66,6 +74,12 @@ export default function JobMutator() { setSpatialFilter(data); }, []); + const handleSpecFilterChange = (key, value) => { + if (!SPEC_FILTERS.map(({ key }) => key).includes(key)) return; + + setSpecFilter({ ...specFilter, [key]: value ? parseFloat(value) : null }); + }; + const isSavingEnabled = () => { return Boolean(notificationAdapterData.length && providerData.length && name); }; @@ -85,6 +99,7 @@ export default function JobMutator() { name, blacklist, spatialFilter, + specFilter, enabled, jobId: jobToBeEdit?.id || null, }); @@ -204,7 +219,7 @@ export default function JobMutator() { @@ -216,6 +231,27 @@ export default function JobMutator() { +
+ {SPEC_FILTERS.map((filter) => ( +
+
{filter.translation}
+ handleSpecFilterChange(filter.key, value)} + /> +
+ ))} +
+ + + diff --git a/ui/src/views/jobs/mutation/JobMutation.less b/ui/src/views/jobs/mutation/JobMutation.less index 2f14cb07..ac6c98b2 100644 --- a/ui/src/views/jobs/mutation/JobMutation.less +++ b/ui/src/views/jobs/mutation/JobMutation.less @@ -3,6 +3,24 @@ float: right; margin-bottom: 1rem; } + + &__specFilter { + display: flex; + gap: 1.5rem; + flex-wrap: wrap; + } + + &__specFilterItem { + display: flex; + flex-direction: column; + gap: 0.5rem; + flex: 1; + min-width: 150px; + } + + &__specFilterLabel { + font-weight: 500; + } } .semi-select-option-list-wrapper { diff --git a/ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx b/ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx index d9c714b4..1925b7ee 100644 --- a/ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx +++ b/ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx @@ -158,11 +158,11 @@ export default function NotificationAdapterMutator({ {uiElement.type === 'boolean' ? (
{ - setValue(selectedAdapter, uiElement, key, checked); - }} - /> + checked={uiElement.value || false} + onChange={(checked) => { + setValue(selectedAdapter, uiElement, key, checked); + }} + /> {uiElement.label}
) : ( diff --git a/ui/src/views/listings/ListingDetail.jsx b/ui/src/views/listings/ListingDetail.jsx index 24bd81f6..39467619 100644 --- a/ui/src/views/listings/ListingDetail.jsx +++ b/ui/src/views/listings/ListingDetail.jsx @@ -31,7 +31,8 @@ import { IconLink, IconStar, IconStarStroked, - IconRealSize, + IconExpand, + IconGridView, } from '@douyinfe/semi-icons'; import maplibregl from 'maplibre-gl'; import 'maplibre-gl/dist/maplibre-gl.css'; @@ -259,6 +260,17 @@ export default function ListingDetail() { if (!listing) return null; const data = [ + { key: 'Price', value: `${listing.price} €`, Icon: }, + { + key: 'Size', + value: listing.size ? `${listing.size} m²` : 'N/A', + Icon: , + }, + { + key: 'Rooms', + value: listing.rooms ? `${listing.rooms} Rooms` : 'N/A', + Icon: , + }, { key: 'Job', value: listing.job_name, @@ -269,12 +281,6 @@ export default function ListingDetail() { value: listing.provider.charAt(0).toUpperCase() + listing.provider.slice(1), Icon: , }, - { key: 'Price', value: `${listing.price} €`, Icon: }, - { - key: 'Size', - value: listing.size ? `${listing.size} m²` : 'N/A', - Icon: , - }, { key: 'Added', value: timeService.format(listing.created_at),