Skip to content

fix: Space filter bug and pulse directory location #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
plur9 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: Space filter bug and pulse directory location #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion config/datacortex.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ server:

# Pulse settings
pulse:
directory: pulses
directory: .datacore/pulses
schedule: manual # daily, weekly, manual

# Graph generation settings
Expand All @@ -31,3 +31,24 @@ graph:
visualization:
node_size_metric: degree # degree, centrality
color_by: type # type, space, cluster

# AI settings
ai:
# === Embeddings ===
# Local embedding model (sentence-transformers)
embedding_model: sentence-transformers/all-mpnet-base-v2

# Content to embed: title + first N chars of content
content_length: 500

# Batch size for embedding computation
batch_size: 32

# === Q&A Synthesis ===
# LLM for answering questions (when running standalone, not in Claude Code)
# Options: claude-3-haiku-20240307, claude-3-5-sonnet-20241022, claude-sonnet-4-20250514
qa_model: claude-3-haiku-20240307

# API key from env: ANTHROPIC_API_KEY or .datacore/env/anthropic.env
# Max tokens for answer generation
qa_max_tokens: 1024
145 changes: 101 additions & 44 deletions frontend/js/app.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,38 @@ class Datacortex {
this.graphView = null;
this.selectedNode = null;
this.pulses = [];
this.filters = {
spaces: new Set(),
types: new Set(),
minDegree: 1,
searchQuery: ''
};

// Simple filter model: sets contain what's INCLUDED (checked)
this.includedSpaces = new Set(); // Empty = show nothing, check to add
this.includedTypes = new Set(); // Empty = show nothing, check to add
this.minDegree = 1;
this.searchQuery = '';

// Track all available options (populated from first API call)
this.allSpaces = new Set();
this.allTypes = new Set();
this.initialLoadDone = false;

// Debounced graph loader
this.debouncedLoadGraph = this.debounce(() => this.loadGraph(), 300);

this.init();
}

debounce(fn, ms) {
let timeout;
return (...args) => {
clearTimeout(timeout);
timeout = setTimeout(() => fn.apply(this, args), ms);
};
}

async init() {
// Initialize graph visualization
this.graphView = new GraphView('#graph', this);

// Load initial data
await this.loadGraph();
// First load - get all data to discover available spaces/types
await this.loadGraph({ initial: true });
await this.loadPulses();
await this.loadTags();

Expand All @@ -39,32 +55,69 @@ class Datacortex {
try {
const queryParams = new URLSearchParams();

if (this.filters.spaces.size > 0) {
queryParams.set('spaces', Array.from(this.filters.spaces).join(','));
}
if (this.filters.types.size > 0) {
queryParams.set('types', Array.from(this.filters.types).join(','));
// Only apply filters after initial load
if (this.initialLoadDone) {
// Send included spaces to API (if not all)
if (this.includedSpaces.size > 0 && this.includedSpaces.size < this.allSpaces.size) {
queryParams.set('spaces', [...this.includedSpaces].join(','));
} else if (this.includedSpaces.size === 0) {
// Nothing selected - show empty graph
this.graph = { nodes: [], links: [], stats: { node_count: 0, edge_count: 0, avg_degree: 0, cluster_count: 0, orphan_count: 0, nodes_by_space: {}, nodes_by_type: {} }};
this.graphView.render({ nodes: [], links: [] });
this.updateStats();
return;
}
// Types are filtered client-side after fetch
}
if (this.filters.minDegree > 0) {
queryParams.set('min_degree', this.filters.minDegree);

if (this.minDegree > 0) {
queryParams.set('min_degree', this.minDegree);
}

const url = `${API_BASE}/graph?${queryParams}`;
console.log('Fetching:', url);
const response = await fetch(url);
this.graph = await response.json();

// Apply search filter client-side
// On initial load, discover all spaces/types and include all by default
if (params.initial || !this.initialLoadDone) {
if (this.graph.stats) {
this.allSpaces = new Set(Object.keys(this.graph.stats.nodes_by_space || {}));
this.allTypes = new Set(Object.keys(this.graph.stats.nodes_by_type || {}));
// Start with NOTHING selected for testing
this.includedSpaces = new Set(); // Empty - user opts in
this.includedTypes = new Set([...this.allTypes]); // All types by default
}
this.initialLoadDone = true;

// Show empty graph initially
this.graphView.render({ nodes: [], links: [] });
this.updateStats();
return;
}

// Apply type filter client-side
let nodes = this.graph.nodes;
let links = this.graph.links;

if (this.filters.searchQuery) {
const query = this.filters.searchQuery.toLowerCase();
if (this.includedTypes.size > 0 && this.includedTypes.size < this.allTypes.size) {
nodes = nodes.filter(n => this.includedTypes.has(n.type));
const nodeIds = new Set(nodes.map(n => n.id));
links = links.filter(l => {
const sourceId = typeof l.source === 'object' ? l.source.id : l.source;
const targetId = typeof l.target === 'object' ? l.target.id : l.target;
return nodeIds.has(sourceId) && nodeIds.has(targetId);
});
}

// Apply search filter client-side
if (this.searchQuery) {
const query = this.searchQuery.toLowerCase();
nodes = nodes.filter(n =>
n.title.toLowerCase().includes(query) ||
(n.tags && n.tags.some(t => t.toLowerCase().includes(query)))
);
const nodeIds = new Set(nodes.map(n => n.id));
// Handle both string IDs (from API) and object refs (after D3 render)
links = links.filter(l => {
const sourceId = typeof l.source === 'object' ? l.source.id : l.source;
const targetId = typeof l.target === 'object' ? l.target.id : l.target;
Expand Down Expand Up @@ -174,62 +227,64 @@ class Datacortex {
content.innerHTML = `
<div class="stat-row">
<span class="stat-label">Nodes</span>
<span class="stat-value">${stats.node_count.toLocaleString()}</span>
<span class="stat-value">${(stats.node_count || 0).toLocaleString()}</span>
</div>
<div class="stat-row">
<span class="stat-label">Edges</span>
<span class="stat-value">${stats.edge_count.toLocaleString()}</span>
<span class="stat-value">${(stats.edge_count || 0).toLocaleString()}</span>
</div>
<div class="stat-row">
<span class="stat-label">Avg Degree</span>
<span class="stat-value">${stats.avg_degree.toFixed(1)}</span>
<span class="stat-value">${(stats.avg_degree || 0).toFixed(1)}</span>
</div>
<div class="stat-row">
<span class="stat-label">Clusters</span>
<span class="stat-value">${stats.cluster_count || '-'}</span>
</div>
<div class="stat-row">
<span class="stat-label">Orphans</span>
<span class="stat-value">${stats.orphan_count}</span>
<span class="stat-value">${stats.orphan_count || 0}</span>
</div>
`;

// Build filter checkboxes from data
this.buildSpaceFilters(stats.nodes_by_space);
this.buildTypeFilters(stats.nodes_by_type);
// Build filter checkboxes - use allSpaces/allTypes for options
this.buildSpaceFilters();
this.buildTypeFilters();
}

buildSpaceFilters(spaceStats) {
buildSpaceFilters() {
const container = document.getElementById('space-filters');
const colors = {
personal: 'var(--space-personal)',
datafund: 'var(--space-datafund)',
datacore: 'var(--space-datacore)'
};

container.innerHTML = Object.entries(spaceStats).map(([space, count]) => `
// Use allSpaces for the options, includedSpaces for checked state
container.innerHTML = [...this.allSpaces].map(space => `
<label class="filter-checkbox">
<input type="checkbox" data-space="${space}" checked>
<input type="checkbox" data-space="${space}" ${this.includedSpaces.has(space) ? 'checked' : ''}>
<span class="color-dot" style="background: ${colors[space] || '#888'}"></span>
${space} (${count})
${space}
</label>
`).join('');

// Add event listeners
container.querySelectorAll('input').forEach(input => {
input.addEventListener('change', () => {
const space = input.dataset.space;
if (input.checked) {
this.filters.spaces.delete(input.dataset.space);
this.includedSpaces.add(space);
} else {
this.filters.spaces.add(input.dataset.space);
this.includedSpaces.delete(space);
}
// Note: spaces filter is inverted (checked = include all)
this.loadGraph();
console.log('Included spaces:', [...this.includedSpaces]);
this.debouncedLoadGraph();
});
});
}

buildTypeFilters(typeStats) {
buildTypeFilters() {
const container = document.getElementById('type-filters');
const colors = {
zettel: 'var(--node-zettel)',
Expand All @@ -239,24 +294,26 @@ class Datacortex {
stub: 'var(--node-stub)'
};

container.innerHTML = Object.entries(typeStats).map(([type, count]) => `
// Use allTypes for options, includedTypes for checked state
container.innerHTML = [...this.allTypes].map(type => `
<label class="filter-checkbox">
<input type="checkbox" data-type="${type}" checked>
<input type="checkbox" data-type="${type}" ${this.includedTypes.has(type) ? 'checked' : ''}>
<span class="color-dot" style="background: ${colors[type] || '#888'}"></span>
${type} (${count})
${type}
</label>
`).join('');

// Add event listeners for type filters
// Add event listeners
container.querySelectorAll('input').forEach(input => {
input.addEventListener('change', () => {
const type = input.dataset.type;
if (input.checked) {
this.filters.types.delete(input.dataset.type);
this.includedTypes.add(type);
} else {
this.filters.types.add(input.dataset.type);
this.includedTypes.delete(type);
}
// Note: types filter is inverted (checked = include, unchecked = exclude)
this.loadGraph();
console.log('Included types:', [...this.includedTypes]);
this.debouncedLoadGraph();
});
});
}
Expand Down Expand Up @@ -368,7 +425,7 @@ class Datacortex {
}

filterByTag(tag) {
this.filters.searchQuery = tag;
this.searchQuery = tag;
document.getElementById('search').value = tag;
this.loadGraph();
}
Expand Down