NestJS - 从 POST 请求获取响应 body 并用另一种方法将其保存到 mongoDB
NestJS - get response body from POST request and save it to mongoDB in another method
在调用一个 POST 请求后，我正在尝试从 1 个响应 body 中获得响应 body（并且我成功了），然后我也想保存它到 Mongo 数据库。但问题是它实际上并没有保存它（可能是因为 Mongoose）并且它没有使用想要的 body，只是保存了“ID”。
这里是controller(装饰器@Post('scrape/:id'))
import { Body, Controller, Get, Post, Param, Res } from '@nestjs/common';
import { CreateBooleanStringDto } from '../dto/handles/create-boolean.dto';
import { CreateRawHandlesDto } from '../dto/handles/createRawHandlesDto';
import { HandlesService } from './handles.service';
import { BooleanString } from './interfaces/booleanString.interface';
import { RawHandles } from './interfaces/rawHandles.interface';
@Controller('booleans')
export class HandlesController {
constructor(private readonly handlesService: HandlesService) {}
@Get()
async findAll(): Promise<BooleanString[]> {
return await this.handlesService.findAll();
}
@Get(':id')
async findOne(@Param('id') id): Promise<BooleanString> {
return await this.handlesService.findOne(id);
}
@Post()
create(@Body() createBooleanStringDto: CreateBooleanStringDto): Promise<BooleanString> {
return this.handlesService.create(createBooleanStringDto)
}
@Post('scrape/:id')
async getBoolean(@Param('id') id, @Body() createRawHandlesDto: CreateRawHandlesDto): Promise<CreateRawHandlesDto> {
let term = await this.handlesService.getBoolean(id);
let handles = await this.handlesService.scrapeHandles([term.value]);
await this.handlesService.saveRawHandles(handles)
return handles;
}
@Post('map/:id')
async mapWriteHandles(@Param('id') id){
let booleanObject = await this.handlesService.getBoolean(id);
console.log(booleanObject)
return await this.handlesService.mapWriteHandles('https://linkedin', booleanObject.groupName)
}
}
服务（方法scrapeHandles returns结果我需要在方法saveRawHandles中保存到Mongo
import { Injectable, Logger} from '@nestjs/common';
import { BooleanString } from './interfaces/booleanString.interface';
import { CreateRawHandlesDto } from '../dto/handles/createRawHandlesDto';
import { Model } from 'mongoose';
import { InjectModel } from '@nestjs/mongoose';
import { writeFile, readFileSync } from 'fs';
import { RawHandles } from './interfaces/rawHandles.interface';
@Injectable()
export class HandlesService {
private readonly logger = new Logger(HandlesService.name);
constructor(
@InjectModel('BooleanString')
private readonly booleanStringModel: Model<BooleanString>,
@InjectModel('RawHandles')
private readonly rawHandlesModel: Model<RawHandles>
) {}
async findAll(): Promise<BooleanString[]> {
return await this.booleanStringModel.find();
}
async findOne(id: string): Promise<BooleanString> {
return await this.booleanStringModel.findOne({ _id: id });
}
async create(booleanString: BooleanString): Promise<BooleanString> {
const newBoolean = new this.booleanStringModel(booleanString);
return await newBoolean.save();
}
async getBoolean(id: string): Promise<BooleanString> {
const booleanObject = await this.booleanStringModel.findOne({ _id: id });
const booleanStringValue = booleanObject.value;
return await this.booleanStringModel.findOne({ _id: id });
}
async scrapeHandles(booleanString): Promise<any> {
const se_scraper = require('se-scraper'); //
const scrape_job = {
search_engine: 'bing',
keywords: booleanString,
num_pages: 50,
};
const results = await se_scraper.scrape({}, scrape_job);
this.logger.log('Puppeter finished scraping handles from Bing')
return results;
}
async saveRawHandles(createRawHandlesDto: CreateRawHandlesDto): Promise<RawHandles> {
const createdRawHandles = new this.rawHandlesModel(createRawHandlesDto);
const dupa = new this.rawHandlesModel
return createdRawHandles.save();
}
async mapWriteHandles(uriToReplace, groupName): Promise<any> {
const fileToMap = require('../output/rawHandlesFromBing.json');
const handles = [];
const mapHandles = (uriToReplace) => {
const scrapedProspect = fileToMap.results.results;
Object.keys(scrapedProspect).forEach((key) => {
const resultPage = scrapedProspect[key];
Object.keys(resultPage).forEach((key) => {
const results = resultPage[key].results;
results.forEach((x) => {
const link = x.link.replace(uriToReplace, '');
handles.push(link);
});
});
});
};
const writeHandlesToFile = (groupName) => {
const obj = { groupName, handles };
const jsonOutput = JSON.stringify(obj);
console.log(obj);
writeFile('handles.json', jsonOutput, (err) => {
if (err) console.log(err);
else console.log('success');
});
};
const main = (uriToReplace, groupName) => {
mapHandles(uriToReplace);
writeHandlesToFile(groupName);
};
}
}
rawHandles 接口
export interface RawHandles {
id?: string,
booleanID: string,
results: any
}
rawHandles DTO
export class CreateRawHandlesDto {
readonly booleanID: string;
readonly results: any;
}
示例 JSON object (results) 我想保存到 rawHandles in [=73] =]
{
"results": {
"Backend AND Python developer": {
"1": {
"num_results": "Wyniki: 7 000 000",
"no_results": false,
"effective_query": "",
"results": [
{
"link": "https://www.pythonistaplanet.com/how-to-become-a-python-backend-developer/#:~:text=To%20become%20a%20Python%20backend%20developer%2C%20first%20of,to%20have%20a%20basic%20understanding%20of%20front-end%20technologies.",
"title": "How To Become A Python Back-End Developer – Pythonista Planet",
"snippet": "",
"visible_link": "",
"rank": 1
},
{
"link": "https://code-maven.com/backend-python-developer",
"title": "Backend Python developer - Junior Python …",
"snippet": "11.03.2020 · A large subset of backend developers write web applications. They need to know the appropriate web framework used at their company. There are many Python-based web frameworks, but the most popular ones are Django and Flask. Learn at least one of them to increase your chances to become a good backend developer. Hint: Flask is much easire to learn.",
"visible_link": "https://code-maven.com/backend-python-developer",
"rank": 2
},
"metadata": {
"scraping_detected": false,
"elapsed_time": "16289",
"ms_per_keyword": "325.78",
"num_requests": 50
}
}
编辑：
我一直在尝试做一些不同但仍然没有成功，我不知道如何将它传递给 saveRawHandles() 方法并使用 Mongoose 将其保存到 Mongo。这是控制器和服务的最新代码
控制器：
@Post('scrape/:id')
async getBoolean(@Param('id') id, @Body() createRawHandlesDto: CreateRawHandlesDto): Promise<CreateRawHandlesDto> {
let term = await this.handlesService.getBoolean(id);
let handles = await this.handlesService.scrapeHandles([term.value]);
let rawHandles = {
booleanID: term.id,
results: handles
}
console.log(rawHandles)
await this.handlesService.saveRawHandles(rawHandles)
return handles;
}
console.log (json object rawHandles)
{
booleanID: '60c8c4a481c9af67bf6c874a',
results: {
results: { 'Backend AND Python developer': [Object] },
metadata: {
scraping_detected: false,
elapsed_time: '15934',
ms_per_keyword: '318.68',
num_requests: 50
}
}
服务
async saveRawHandles(createRawHandlesDto: CreateRawHandlesDto): Promise<RawHandles> {
const createdRawHandles = new this.rawHandlesModel(createRawHandlesDto);
return createdRawHandles.save();
}
这是来自 Mongodb in cloud 的屏幕截图，基本上它只是添加新的 objects 但不是按照模式
MongoDB CLoud screenshoot
终于解决了。朋友建议我不要使用 Mongoose，因为它是某种 TypeORM (ODM)，而只使用 Mongo，它是 API。所以我所做的是使用 Provider:
mondo.module.ts
import { Module } from '@nestjs/common';
import { MongoClient, Db } from 'mongodb';
@Module({
providers: [
{
provide: 'DATABASE_CONNECTION',
useFactory: async (): Promise<Db> => {
try {
const client = await MongoClient.connect(
process.env.MONGO_CONNECTION,
{
useUnifiedTopology: true,
auth: {
user: process.env.MONGO_USER,
password: process.env.MONGO_PASSWORD,
},
},
);
return client.db();
} catch (e) {
throw e;
}
},
},
],
exports: ['DATABASE_CONNECTION'],
})
export class DatabaseModule {}
然后将其导入到您的模块中。
handless.module.ts
import { DatabaseModule } from '../core/mongo/mongo.module';
@Module({
imports: [
MongooseModule.forFeature([{ name: 'BooleanString', schema: BooleanStringSchema }]),
DatabaseModule
],
controllers: [HandlesController],
providers: [HandlesService],
})
export class HandlesModule {}
现在我们可以在我们的服务中使用它了。这是我完成的一个最小示例，它在我这边工作正常，如我所愿
handless.service.ts
import { Injectable, Logger, Inject } from '@nestjs/common';
import { BooleanString } from './interfaces/booleanString.interface';
import { Model } from 'mongoose';
import { Db, Collection } from 'mongodb';
import { InjectModel } from '@nestjs/mongoose';
import { writeFile } from 'fs';
@Injectable()
export class HandlesService {
private readonly logger = new Logger(HandlesService.name);
private handlesCollection: Collection<any>;
constructor(
@InjectModel('BooleanString')
private readonly booleanStringModel: Model<BooleanString>,
@Inject('DATABASE_CONNECTION')
private db: Db,
) {
this.handlesCollection = this.db.collection('rawHandles');
}
async scrapeHandles(booleanString): Promise<any> {
const se_scraper = require('se-scraper'); //
const scrape_job = {
search_engine: 'bing',
keywords: booleanString,
num_pages: 50,
};
const results = await se_scraper.scrape({}, scrape_job);
this.logger.log('Puppeter finished scraping handles from Bing')
return results;
}
async saveRawHandles(results): Promise<any> {
return await this.handlesCollection.insertOne(results);
}
和控制器。
handless.controller.ts
import { Body, Controller, Get, Post, Param, Res } from '@nestjs/common';
import { CreateBooleanStringDto } from '../dto/handles/create-boolean.dto';
import { HandlesService } from './handles.service';
import { BooleanString } from './interfaces/booleanString.interface';
@Controller('booleans')
export class HandlesController {
constructor(private readonly handlesService: HandlesService) {}
@Post('scrape/:id')
async getBoolean(@Param('id') id): Promise<any> {
let term = await this.handlesService.getBoolean(id);
let handles = await this.handlesService.scrapeHandles([term.value]);
let results = {
booleanSearchID: term.id,
results: handles
}
await this.handlesService.saveRawHandles(results);
return handles;
}
}
在调用一个 POST 请求后，我正在尝试从 1 个响应 body 中获得响应 body（并且我成功了），然后我也想保存它到 Mongo 数据库。但问题是它实际上并没有保存它（可能是因为 Mongoose）并且它没有使用想要的 body，只是保存了“ID”。
这里是controller(装饰器@Post('scrape/:id'))
import { Body, Controller, Get, Post, Param, Res } from '@nestjs/common';
import { CreateBooleanStringDto } from '../dto/handles/create-boolean.dto';
import { CreateRawHandlesDto } from '../dto/handles/createRawHandlesDto';
import { HandlesService } from './handles.service';
import { BooleanString } from './interfaces/booleanString.interface';
import { RawHandles } from './interfaces/rawHandles.interface';
@Controller('booleans')
export class HandlesController {
constructor(private readonly handlesService: HandlesService) {}
@Get()
async findAll(): Promise<BooleanString[]> {
return await this.handlesService.findAll();
}
@Get(':id')
async findOne(@Param('id') id): Promise<BooleanString> {
return await this.handlesService.findOne(id);
}
@Post()
create(@Body() createBooleanStringDto: CreateBooleanStringDto): Promise<BooleanString> {
return this.handlesService.create(createBooleanStringDto)
}
@Post('scrape/:id')
async getBoolean(@Param('id') id, @Body() createRawHandlesDto: CreateRawHandlesDto): Promise<CreateRawHandlesDto> {
let term = await this.handlesService.getBoolean(id);
let handles = await this.handlesService.scrapeHandles([term.value]);
await this.handlesService.saveRawHandles(handles)
return handles;
}
@Post('map/:id')
async mapWriteHandles(@Param('id') id){
let booleanObject = await this.handlesService.getBoolean(id);
console.log(booleanObject)
return await this.handlesService.mapWriteHandles('https://linkedin', booleanObject.groupName)
}
}
服务（方法scrapeHandles returns结果我需要在方法saveRawHandles中保存到Mongo
import { Injectable, Logger} from '@nestjs/common';
import { BooleanString } from './interfaces/booleanString.interface';
import { CreateRawHandlesDto } from '../dto/handles/createRawHandlesDto';
import { Model } from 'mongoose';
import { InjectModel } from '@nestjs/mongoose';
import { writeFile, readFileSync } from 'fs';
import { RawHandles } from './interfaces/rawHandles.interface';
@Injectable()
export class HandlesService {
private readonly logger = new Logger(HandlesService.name);
constructor(
@InjectModel('BooleanString')
private readonly booleanStringModel: Model<BooleanString>,
@InjectModel('RawHandles')
private readonly rawHandlesModel: Model<RawHandles>
) {}
async findAll(): Promise<BooleanString[]> {
return await this.booleanStringModel.find();
}
async findOne(id: string): Promise<BooleanString> {
return await this.booleanStringModel.findOne({ _id: id });
}
async create(booleanString: BooleanString): Promise<BooleanString> {
const newBoolean = new this.booleanStringModel(booleanString);
return await newBoolean.save();
}
async getBoolean(id: string): Promise<BooleanString> {
const booleanObject = await this.booleanStringModel.findOne({ _id: id });
const booleanStringValue = booleanObject.value;
return await this.booleanStringModel.findOne({ _id: id });
}
async scrapeHandles(booleanString): Promise<any> {
const se_scraper = require('se-scraper'); //
const scrape_job = {
search_engine: 'bing',
keywords: booleanString,
num_pages: 50,
};
const results = await se_scraper.scrape({}, scrape_job);
this.logger.log('Puppeter finished scraping handles from Bing')
return results;
}
async saveRawHandles(createRawHandlesDto: CreateRawHandlesDto): Promise<RawHandles> {
const createdRawHandles = new this.rawHandlesModel(createRawHandlesDto);
const dupa = new this.rawHandlesModel
return createdRawHandles.save();
}
async mapWriteHandles(uriToReplace, groupName): Promise<any> {
const fileToMap = require('../output/rawHandlesFromBing.json');
const handles = [];
const mapHandles = (uriToReplace) => {
const scrapedProspect = fileToMap.results.results;
Object.keys(scrapedProspect).forEach((key) => {
const resultPage = scrapedProspect[key];
Object.keys(resultPage).forEach((key) => {
const results = resultPage[key].results;
results.forEach((x) => {
const link = x.link.replace(uriToReplace, '');
handles.push(link);
});
});
});
};
const writeHandlesToFile = (groupName) => {
const obj = { groupName, handles };
const jsonOutput = JSON.stringify(obj);
console.log(obj);
writeFile('handles.json', jsonOutput, (err) => {
if (err) console.log(err);
else console.log('success');
});
};
const main = (uriToReplace, groupName) => {
mapHandles(uriToReplace);
writeHandlesToFile(groupName);
};
}
}
rawHandles 接口
export interface RawHandles {
id?: string,
booleanID: string,
results: any
}
rawHandles DTO
export class CreateRawHandlesDto {
readonly booleanID: string;
readonly results: any;
}
示例 JSON object (results) 我想保存到 rawHandles in [=73] =]
{
"results": {
"Backend AND Python developer": {
"1": {
"num_results": "Wyniki: 7 000 000",
"no_results": false,
"effective_query": "",
"results": [
{
"link": "https://www.pythonistaplanet.com/how-to-become-a-python-backend-developer/#:~:text=To%20become%20a%20Python%20backend%20developer%2C%20first%20of,to%20have%20a%20basic%20understanding%20of%20front-end%20technologies.",
"title": "How To Become A Python Back-End Developer – Pythonista Planet",
"snippet": "",
"visible_link": "",
"rank": 1
},
{
"link": "https://code-maven.com/backend-python-developer",
"title": "Backend Python developer - Junior Python …",
"snippet": "11.03.2020 · A large subset of backend developers write web applications. They need to know the appropriate web framework used at their company. There are many Python-based web frameworks, but the most popular ones are Django and Flask. Learn at least one of them to increase your chances to become a good backend developer. Hint: Flask is much easire to learn.",
"visible_link": "https://code-maven.com/backend-python-developer",
"rank": 2
},
"metadata": {
"scraping_detected": false,
"elapsed_time": "16289",
"ms_per_keyword": "325.78",
"num_requests": 50
}
}
编辑：
我一直在尝试做一些不同但仍然没有成功，我不知道如何将它传递给 saveRawHandles() 方法并使用 Mongoose 将其保存到 Mongo。这是控制器和服务的最新代码 控制器：
@Post('scrape/:id')
async getBoolean(@Param('id') id, @Body() createRawHandlesDto: CreateRawHandlesDto): Promise<CreateRawHandlesDto> {
let term = await this.handlesService.getBoolean(id);
let handles = await this.handlesService.scrapeHandles([term.value]);
let rawHandles = {
booleanID: term.id,
results: handles
}
console.log(rawHandles)
await this.handlesService.saveRawHandles(rawHandles)
return handles;
}
console.log (json object rawHandles)
{
booleanID: '60c8c4a481c9af67bf6c874a',
results: {
results: { 'Backend AND Python developer': [Object] },
metadata: {
scraping_detected: false,
elapsed_time: '15934',
ms_per_keyword: '318.68',
num_requests: 50
}
}
服务
async saveRawHandles(createRawHandlesDto: CreateRawHandlesDto): Promise<RawHandles> {
const createdRawHandles = new this.rawHandlesModel(createRawHandlesDto);
return createdRawHandles.save();
}
这是来自 Mongodb in cloud 的屏幕截图，基本上它只是添加新的 objects 但不是按照模式 MongoDB CLoud screenshoot
终于解决了。朋友建议我不要使用 Mongoose，因为它是某种 TypeORM (ODM)，而只使用 Mongo，它是 API。所以我所做的是使用 Provider:
mondo.module.ts
import { Module } from '@nestjs/common';
import { MongoClient, Db } from 'mongodb';
@Module({
providers: [
{
provide: 'DATABASE_CONNECTION',
useFactory: async (): Promise<Db> => {
try {
const client = await MongoClient.connect(
process.env.MONGO_CONNECTION,
{
useUnifiedTopology: true,
auth: {
user: process.env.MONGO_USER,
password: process.env.MONGO_PASSWORD,
},
},
);
return client.db();
} catch (e) {
throw e;
}
},
},
],
exports: ['DATABASE_CONNECTION'],
})
export class DatabaseModule {}
然后将其导入到您的模块中。 handless.module.ts
import { DatabaseModule } from '../core/mongo/mongo.module';
@Module({
imports: [
MongooseModule.forFeature([{ name: 'BooleanString', schema: BooleanStringSchema }]),
DatabaseModule
],
controllers: [HandlesController],
providers: [HandlesService],
})
export class HandlesModule {}
现在我们可以在我们的服务中使用它了。这是我完成的一个最小示例，它在我这边工作正常，如我所愿 handless.service.ts
import { Injectable, Logger, Inject } from '@nestjs/common';
import { BooleanString } from './interfaces/booleanString.interface';
import { Model } from 'mongoose';
import { Db, Collection } from 'mongodb';
import { InjectModel } from '@nestjs/mongoose';
import { writeFile } from 'fs';
@Injectable()
export class HandlesService {
private readonly logger = new Logger(HandlesService.name);
private handlesCollection: Collection<any>;
constructor(
@InjectModel('BooleanString')
private readonly booleanStringModel: Model<BooleanString>,
@Inject('DATABASE_CONNECTION')
private db: Db,
) {
this.handlesCollection = this.db.collection('rawHandles');
}
async scrapeHandles(booleanString): Promise<any> {
const se_scraper = require('se-scraper'); //
const scrape_job = {
search_engine: 'bing',
keywords: booleanString,
num_pages: 50,
};
const results = await se_scraper.scrape({}, scrape_job);
this.logger.log('Puppeter finished scraping handles from Bing')
return results;
}
async saveRawHandles(results): Promise<any> {
return await this.handlesCollection.insertOne(results);
}
和控制器。 handless.controller.ts
import { Body, Controller, Get, Post, Param, Res } from '@nestjs/common';
import { CreateBooleanStringDto } from '../dto/handles/create-boolean.dto';
import { HandlesService } from './handles.service';
import { BooleanString } from './interfaces/booleanString.interface';
@Controller('booleans')
export class HandlesController {
constructor(private readonly handlesService: HandlesService) {}
@Post('scrape/:id')
async getBoolean(@Param('id') id): Promise<any> {
let term = await this.handlesService.getBoolean(id);
let handles = await this.handlesService.scrapeHandles([term.value]);
let results = {
booleanSearchID: term.id,
results: handles
}
await this.handlesService.saveRawHandles(results);
return handles;
}
}