aliyrlmz•2mo ago

Live company data researching/streaming as a service.

Hello, ive got an idea to build scraper based on google/yandex/linked in search results to gather spesific categoried company informations as a service on my country. Ive implemented convex but even while on development i gapped through database bandwith. Ive really really want to use convex but i cant affort since its startup and i do not have users for like 2-3 weeks on start. Is there a way to fix it ? Ill make sure to answer any question about data/schema/functions. Nextjs app register/login via clerk. Add research token on registering via clerk on convex db. User starts research with credits. My backend server listens tasks records for pending requests. Claims task to work on it and stream results on convex database. Nextjs app listens tast results via convex function and serves to customer. Mainly shown on video at 40-50 secs. Im really looking forward to any help...

1 Reply

aliyrlmzOP•2mo ago

Also trial live on https://kimbusirket.com/

export const pushResult = mutation({
  args: { 
    taskId: v.id("tasks"), 
    batchId: v.string(),
    data: v.object({
      places: v.array(v.any()),
      credits: v.number(),
    }),
    companyCount: v.optional(v.number()),
  },
  handler: async (ctx, args) => {
    await ctx.db.insert("results", {
      taskId: args.taskId,
      batchId: args.batchId,
      data: args.data,
      companyCount: args.companyCount ?? 0,
      timestamp: Date.now(),
    });
    // Task tablosunda companyCount'u güncelle
    const task = await ctx.db.get(args.taskId);
    if (task) {
      const allResults = await ctx.db.query("results").filter(q => q.eq(q.field("taskId"), args.taskId)).collect();
      const totalCompanyCount = allResults.reduce((sum, r) => sum + (r.companyCount ?? 0), 0);
      await ctx.db.patch(args.taskId, { companyCount: totalCompanyCount });
    }
    return { success: true };
  },
});

export const pushResult = mutation({
  args: { 
    taskId: v.id("tasks"), 
    batchId: v.string(),
    data: v.object({
      places: v.array(v.any()),
      credits: v.number(),
    }),
    companyCount: v.optional(v.number()),
  },
  handler: async (ctx, args) => {
    await ctx.db.insert("results", {
      taskId: args.taskId,
      batchId: args.batchId,
      data: args.data,
      companyCount: args.companyCount ?? 0,
      timestamp: Date.now(),
    });
    // Task tablosunda companyCount'u güncelle
    const task = await ctx.db.get(args.taskId);
    if (task) {
      const allResults = await ctx.db.query("results").filter(q => q.eq(q.field("taskId"), args.taskId)).collect();
      const totalCompanyCount = allResults.reduce((sum, r) => sum + (r.companyCount ?? 0), 0);
      await ctx.db.patch(args.taskId, { companyCount: totalCompanyCount });
    }
    return { success: true };
  },
});

export const listResults = query({
  args: {
    taskId: v.id("tasks"),
    limit: v.optional(v.number()),
    skip: v.optional(v.number()),
    searchTerm: v.optional(v.string()),
    districtName: v.optional(v.string()), // id yerine isim!
    sortField: v.optional(v.union(v.literal("title"), v.literal("district"), v.literal("address"))),
    sortOrder: v.optional(v.union(v.literal("asc"), v.literal("desc"))),
  },
  handler: async (ctx, args) => {
    let resultsQuery = ctx.db
      .query("results")
      .filter(q => q.eq(q.field("taskId"), args.taskId))
      .order("asc");
    let results = await resultsQuery.collect();
    let allPlaces = results.flatMap(r => r.data.places.map((p: any) => ({ ...p })));

    // districtStats: isim bazlı
    const districtMap = new Map();
    allPlaces.forEach((p: any) => {
      const name = (p.district || "Bilinmeyen İlçe").trim();
      if (!districtMap.has(name)) districtMap.set(name, 0);
      districtMap.set(name, districtMap.get(name) + 1);
    });
    const districtStats = Array.from(districtMap.entries()).map(([name, count]) => ({ name, count }));

    // Filtre: districtName
    let filteredPlaces = allPlaces;
    if (args.districtName && args.districtName !== "Tümü") {
      filteredPlaces = filteredPlaces.filter(p => (p.district || "").trim() === args.districtName);
    }
    // Filtre: searchTerm
    if (args.searchTerm && args.searchTerm.trim() !== "") {
      const term = args.searchTerm.toLowerCase();
      );
    }
    // Sıralama
    if (args.sortField) {
     
    }
    // Pagination
    const skip = args.skip ?? 0;
    const limit = args.limit ?? 10;
    const pagedPlaces = filteredPlaces.slice(skip, skip + limit);
    const total = filteredPlaces.length;
    return { places: pagedPlaces, total, districtStats };
  },
});

export const listResults = query({
  args: {
    taskId: v.id("tasks"),
    limit: v.optional(v.number()),
    skip: v.optional(v.number()),
    searchTerm: v.optional(v.string()),
    districtName: v.optional(v.string()), // id yerine isim!
    sortField: v.optional(v.union(v.literal("title"), v.literal("district"), v.literal("address"))),
    sortOrder: v.optional(v.union(v.literal("asc"), v.literal("desc"))),
  },
  handler: async (ctx, args) => {
    let resultsQuery = ctx.db
      .query("results")
      .filter(q => q.eq(q.field("taskId"), args.taskId))
      .order("asc");
    let results = await resultsQuery.collect();
    let allPlaces = results.flatMap(r => r.data.places.map((p: any) => ({ ...p })));

    // districtStats: isim bazlı
    const districtMap = new Map();
    allPlaces.forEach((p: any) => {
      const name = (p.district || "Bilinmeyen İlçe").trim();
      if (!districtMap.has(name)) districtMap.set(name, 0);
      districtMap.set(name, districtMap.get(name) + 1);
    });
    const districtStats = Array.from(districtMap.entries()).map(([name, count]) => ({ name, count }));

    // Filtre: districtName
    let filteredPlaces = allPlaces;
    if (args.districtName && args.districtName !== "Tümü") {
      filteredPlaces = filteredPlaces.filter(p => (p.district || "").trim() === args.districtName);
    }
    // Filtre: searchTerm
    if (args.searchTerm && args.searchTerm.trim() !== "") {
      const term = args.searchTerm.toLowerCase();
      );
    }
    // Sıralama
    if (args.sortField) {
     
    }
    // Pagination
    const skip = args.skip ?? 0;
    const limit = args.limit ?? 10;
    const pagedPlaces = filteredPlaces.slice(skip, skip + limit);
    const total = filteredPlaces.length;
    return { places: pagedPlaces, total, districtStats };
  },
});

this two functions basicly take all my bandwith one registering companies/results. one shows it to users. Fixed with some improvements on functions and moved to convex on local server instead.

Live company data researching/streaming as a service.

Did you find this page helpful?