如何将多行json对象转化为sqltable结构化数据？
How to transform multiple rows of json objects into sql table structured data?
我使用 SSIS 中的脚本任务创建了 Web API，该脚本任务使用本教程将 JSON 格式的天气数据检索到 SQL 数据库 table：Weather API SSIS。在教程示例中，只有一组坐标被使用，它在数据库中为我们提供了一行 table 和 JSON 对象。
#region Namespaces
using System;
using System.Data;
using Microsoft.SqlServer.Dts.Runtime;
using System.Windows.Forms;
using System.Data.SqlClient;
using System.Net;
#endregion
namespace ST_6f60bececd8f4f94afaf758869590918
{
/// <summary>
/// ScriptMain is the entry point class of the script. Do not change the name, attributes,
/// or parent of this class.
/// </summary>
[Microsoft.SqlServer.Dts.Tasks.ScriptTask.SSISScriptTaskEntryPointAttribute]
public partial class ScriptMain : Microsoft.SqlServer.Dts.Tasks.ScriptTask.VSTARTScriptObjectModelBase
{
#region Help: Using Integration Services variables and parameters in a script
/* To use a variable in this script, first ensure that the variable has been added to
* either the list contained in the ReadOnlyVariables property or the list contained in
* the ReadWriteVariables property of this script task, according to whether or not your
* code needs to write to the variable. To add the variable, save this script, close this instance of
* Visual Studio, and update the ReadOnlyVariables and
* ReadWriteVariables properties in the Script Transformation Editor window.
* To use a parameter in this script, follow the same steps. Parameters are always read-only.
*
* Example of reading from a variable:
* DateTime startTime = (DateTime) Dts.Variables["System::StartTime"].Value;
*
* Example of writing to a variable:
* Dts.Variables["User::myStringVariable"].Value = "new value";
*
* Example of reading from a package parameter:
* int batchId = (int) Dts.Variables["$Package::batchId"].Value;
*
* Example of reading from a project parameter:
* int batchId = (int) Dts.Variables["$Project::batchId"].Value;
*
* Example of reading from a sensitive project parameter:
* int batchId = (int) Dts.Variables["$Project::batchId"].GetSensitiveValue();
* */
#endregion
#region Help: Firing Integration Services events from a script
/* This script task can fire events for logging purposes.
*
* Example of firing an error event:
* Dts.Events.FireError(18, "Process Values", "Bad value", "", 0);
*
* Example of firing an information event:
* Dts.Events.FireInformation(3, "Process Values", "Processing has started", "", 0, ref fireAgain)
*
* Example of firing a warning event:
* Dts.Events.FireWarning(14, "Process Values", "No values received for input", "", 0);
* */
#endregion
#region Help: Using Integration Services connection managers in a script
/* Some types of connection managers can be used in this script task. See the topic
* "Working with Connection Managers Programatically" for details.
*
* Example of using an ADO.Net connection manager:
* object rawConnection = Dts.Connections["Sales DB"].AcquireConnection(Dts.Transaction);
* SqlConnection myADONETConnection = (SqlConnection)rawConnection;
* //Use the connection in some code here, then release the connection
* Dts.Connections["Sales DB"].ReleaseConnection(rawConnection);
*
* Example of using a File connection manager
* object rawConnection = Dts.Connections["Prices.zip"].AcquireConnection(Dts.Transaction);
* string filePath = (string)rawConnection;
* //Use the connection in some code here, then release the connection
* Dts.Connections["Prices.zip"].ReleaseConnection(rawConnection);
* */
#endregion
/// <summary>
/// This method is called when this script task executes in the control flow.
/// Before returning from this method, set the value of Dts.TaskResult to indicate success or failure.
/// To open Help, press F1.
/// </summary>
public void Main()
{
string Longitude = (string)Dts.Variables["User::Longitude"].Value.ToString();
string Latitude = (string)Dts.Variables["User::Latitude"].Value.ToString();
string municipality = (string)Dts.Variables["User::Municipality"].Value.ToString();
MessageBox.Show("Longitude:" + Longitude + ", Latitude:" + Latitude);
var url = @"https://api.met.no/weatherapi/locationforecast/2.0/compact.json?lat="+Latitude+"&lon="+Longitude+"";
System.Net.ServicePointManager.DefaultConnectionLimit = int.MaxValue;
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
req.UseDefaultCredentials = true;
req.Proxy.Credentials = System.Net.CredentialCache.DefaultCredentials;
var syncClient = new WebClient();
syncClient.Headers.Add("user-agent", "acmeweathersite.com support@acmeweathersite.com");
var content = syncClient.DownloadString(url);
string connectionString = "Data Source=localhost;Initial Catalog=Weather;Integrated Security=True;";
using (SqlConnection conn = new SqlConnection(connectionString))
{
SqlCommand Storproc =
new SqlCommand(@"INSERT INTO [dbo].[Weather] (JSONData)
select @JSONData", conn);
Storproc.Parameters.AddWithValue("@JSONData", content.ToString());
conn.Open();
Storproc.ExecuteNonQuery();
conn.Close();
}
// TODO: Add your code here
Dts.TaskResult = (int)ScriptResults.Success;
}
#region ScriptResults declaration
/// <summary>
/// This enum provides a convenient shorthand within the scope of this class for setting the
/// result of the script.
///
/// This code was generated automatically.
/// </summary>
enum ScriptResults
{
Success = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Success,
Failure = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Failure
};
#endregion
}
}
之后，我将从上面发布的脚本任务代码中检索到的 JSON 对象插入到 sql 数据 table 格式的新 table 中:
declare @JSON nvarchar(max)
select @JSON = [JSONData]
FROM [Weather].[dbo].[Weather]
;with cteTemp as
(SELECT *
FROM OPENJSON(@json, '$.properties.timeseries')
WITH (
time datetime '$.time',
air_temperature nvarchar(50) '$.data.instant.details.air_temperature',
wind_speed nvarchar(50) '$.data.instant.details.wind_speed',
precipitation_amount_next_1_hour nvarchar(50) '$.data.next_1_hours.details.precipitation_amount',
symbol_code_next_1_hour nvarchar(50) '$.data.next_1_hours.summary.symbol_code',
precipitation_amount_next_6_hour nvarchar(50) '$.data.next_6_hours.details.precipitation_amount',
symbol_code_next_6_hour nvarchar(50) '$.data.next_6_hours.summary.symbol_code'
)
)
insert into [dbo].[WeatherByHour]([time],[air_temperature],wind_speed,precipitation_amount_next_1_hour,symbol_code_next_1_hour,precipitation_amount_next_6_hour,symbol_code_next_6_hour)
select [time],
[air_temperature],
wind_speed,
precipitation_amount_next_1_hour,
symbol_code_next_1_hour,
precipitation_amount_next_6_hour,
symbol_code_next_6_hour
from cteTemp;
现在的下一步是我已经创建了 table 具有多个坐标，我将在我的脚本任务中将其用作参数变量并添加一个 foreach 循环：
现在这将给我 3 行 JSON 对象，每个对象对应我的坐标 table 中存在的每组坐标，在 URL [=60] 中用作可变参数=]:
我现在遇到的问题是，当我将 JSON 对象插入我的 WeatherByHour table 时，它只使用第一行 JSON 对象的第一组坐标在我的例子中，它给了我 83 行，但理想情况下我希望它产生 83 + 83 + 83 = 249 行。这里最好的解决方案是什么，在脚本任务中创建一个大的 JSON 数组对象，或者以某种方式遍历 Weather table 中的所有 JSON 对象行并将它们插入 WeatherByHour table ?
这些是我的示例中使用的所有 table：
USE [Weather]
GO
/****** Object: Table [dbo].[Weather] Script Date: 2021-03-14 13:55:44 ******/
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
CREATE TABLE [dbo].[Weather](
[JSONData] [nvarchar](max) NOT NULL
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
USE [Weather]
GO
/****** Object: Table [dbo].[WeatherByHour] Script Date: 2021-03-14 13:56:09 ******/
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
CREATE TABLE [dbo].[WeatherByHour](
[id] [int] IDENTITY(1,1) NOT NULL,
[time] [datetime] NULL,
[air_temperature] [nvarchar](50) NULL,
[wind_speed] [nvarchar](50) NULL,
[precipitation_amount_next_1_hour] [nvarchar](50) NULL,
[symbol_code_next_1_hour] [nvarchar](50) NULL,
[precipitation_amount_next_6_hour] [nvarchar](50) NULL,
[symbol_code_next_6_hour] [nvarchar](50) NULL,
CONSTRAINT [PK_WeatherByHour] PRIMARY KEY CLUSTERED
(
[id] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY]
GO
create table Coordinates(
Municipality nvarchar(50),
Latitide nvarchar(50),
Longitude nvarchar(50)
)
INSERT INTO Coordinates (Municipality, Latitide, Longitude)
VALUES (114, 59.5166667, 17.9),
(115, 59.5833333, 18.2),
(117, 59.5, 18.45)
您可以使用交叉应用将时间序列转换为 json 行。
对于上面的示例 [Weather].[dbo].[Weather] table，您最终将得到 3（无坐标）x N（每个坐标的时间序列）。
SELECT s.value,
--JSON_VALUE(w.JSONData,'$.geometry.coordinates[1]') AS [latitude],
--JSON_VALUE(w.JSONData,'$.geometry.coordinates[0]') AS [longitude],
CAST(JSON_VALUE(s.value, '$.time') AS datetime) AS [time],
JSON_VALUE(s.value, '$.data.instant.details.air_temperature') as [air_temperature],
JSON_VALUE(s.value, '$.data.next_1_hours.details.precipitation_amount') as [precipitation_amount_next_1_hour]
FROM [Weather].[dbo].[Weather] w
CROSS APPLY OPENJSON(w.JSONData,'$.properties.timeseries') s
我使用 SSIS 中的脚本任务创建了 Web API，该脚本任务使用本教程将 JSON 格式的天气数据检索到 SQL 数据库 table：Weather API SSIS。在教程示例中，只有一组坐标被使用，它在数据库中为我们提供了一行 table 和 JSON 对象。
#region Namespaces
using System;
using System.Data;
using Microsoft.SqlServer.Dts.Runtime;
using System.Windows.Forms;
using System.Data.SqlClient;
using System.Net;
#endregion
namespace ST_6f60bececd8f4f94afaf758869590918
{
/// <summary>
/// ScriptMain is the entry point class of the script. Do not change the name, attributes,
/// or parent of this class.
/// </summary>
[Microsoft.SqlServer.Dts.Tasks.ScriptTask.SSISScriptTaskEntryPointAttribute]
public partial class ScriptMain : Microsoft.SqlServer.Dts.Tasks.ScriptTask.VSTARTScriptObjectModelBase
{
#region Help: Using Integration Services variables and parameters in a script
/* To use a variable in this script, first ensure that the variable has been added to
* either the list contained in the ReadOnlyVariables property or the list contained in
* the ReadWriteVariables property of this script task, according to whether or not your
* code needs to write to the variable. To add the variable, save this script, close this instance of
* Visual Studio, and update the ReadOnlyVariables and
* ReadWriteVariables properties in the Script Transformation Editor window.
* To use a parameter in this script, follow the same steps. Parameters are always read-only.
*
* Example of reading from a variable:
* DateTime startTime = (DateTime) Dts.Variables["System::StartTime"].Value;
*
* Example of writing to a variable:
* Dts.Variables["User::myStringVariable"].Value = "new value";
*
* Example of reading from a package parameter:
* int batchId = (int) Dts.Variables["$Package::batchId"].Value;
*
* Example of reading from a project parameter:
* int batchId = (int) Dts.Variables["$Project::batchId"].Value;
*
* Example of reading from a sensitive project parameter:
* int batchId = (int) Dts.Variables["$Project::batchId"].GetSensitiveValue();
* */
#endregion
#region Help: Firing Integration Services events from a script
/* This script task can fire events for logging purposes.
*
* Example of firing an error event:
* Dts.Events.FireError(18, "Process Values", "Bad value", "", 0);
*
* Example of firing an information event:
* Dts.Events.FireInformation(3, "Process Values", "Processing has started", "", 0, ref fireAgain)
*
* Example of firing a warning event:
* Dts.Events.FireWarning(14, "Process Values", "No values received for input", "", 0);
* */
#endregion
#region Help: Using Integration Services connection managers in a script
/* Some types of connection managers can be used in this script task. See the topic
* "Working with Connection Managers Programatically" for details.
*
* Example of using an ADO.Net connection manager:
* object rawConnection = Dts.Connections["Sales DB"].AcquireConnection(Dts.Transaction);
* SqlConnection myADONETConnection = (SqlConnection)rawConnection;
* //Use the connection in some code here, then release the connection
* Dts.Connections["Sales DB"].ReleaseConnection(rawConnection);
*
* Example of using a File connection manager
* object rawConnection = Dts.Connections["Prices.zip"].AcquireConnection(Dts.Transaction);
* string filePath = (string)rawConnection;
* //Use the connection in some code here, then release the connection
* Dts.Connections["Prices.zip"].ReleaseConnection(rawConnection);
* */
#endregion
/// <summary>
/// This method is called when this script task executes in the control flow.
/// Before returning from this method, set the value of Dts.TaskResult to indicate success or failure.
/// To open Help, press F1.
/// </summary>
public void Main()
{
string Longitude = (string)Dts.Variables["User::Longitude"].Value.ToString();
string Latitude = (string)Dts.Variables["User::Latitude"].Value.ToString();
string municipality = (string)Dts.Variables["User::Municipality"].Value.ToString();
MessageBox.Show("Longitude:" + Longitude + ", Latitude:" + Latitude);
var url = @"https://api.met.no/weatherapi/locationforecast/2.0/compact.json?lat="+Latitude+"&lon="+Longitude+"";
System.Net.ServicePointManager.DefaultConnectionLimit = int.MaxValue;
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
req.UseDefaultCredentials = true;
req.Proxy.Credentials = System.Net.CredentialCache.DefaultCredentials;
var syncClient = new WebClient();
syncClient.Headers.Add("user-agent", "acmeweathersite.com support@acmeweathersite.com");
var content = syncClient.DownloadString(url);
string connectionString = "Data Source=localhost;Initial Catalog=Weather;Integrated Security=True;";
using (SqlConnection conn = new SqlConnection(connectionString))
{
SqlCommand Storproc =
new SqlCommand(@"INSERT INTO [dbo].[Weather] (JSONData)
select @JSONData", conn);
Storproc.Parameters.AddWithValue("@JSONData", content.ToString());
conn.Open();
Storproc.ExecuteNonQuery();
conn.Close();
}
// TODO: Add your code here
Dts.TaskResult = (int)ScriptResults.Success;
}
#region ScriptResults declaration
/// <summary>
/// This enum provides a convenient shorthand within the scope of this class for setting the
/// result of the script.
///
/// This code was generated automatically.
/// </summary>
enum ScriptResults
{
Success = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Success,
Failure = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Failure
};
#endregion
}
}
之后，我将从上面发布的脚本任务代码中检索到的 JSON 对象插入到 sql 数据 table 格式的新 table 中:
declare @JSON nvarchar(max)
select @JSON = [JSONData]
FROM [Weather].[dbo].[Weather]
;with cteTemp as
(SELECT *
FROM OPENJSON(@json, '$.properties.timeseries')
WITH (
time datetime '$.time',
air_temperature nvarchar(50) '$.data.instant.details.air_temperature',
wind_speed nvarchar(50) '$.data.instant.details.wind_speed',
precipitation_amount_next_1_hour nvarchar(50) '$.data.next_1_hours.details.precipitation_amount',
symbol_code_next_1_hour nvarchar(50) '$.data.next_1_hours.summary.symbol_code',
precipitation_amount_next_6_hour nvarchar(50) '$.data.next_6_hours.details.precipitation_amount',
symbol_code_next_6_hour nvarchar(50) '$.data.next_6_hours.summary.symbol_code'
)
)
insert into [dbo].[WeatherByHour]([time],[air_temperature],wind_speed,precipitation_amount_next_1_hour,symbol_code_next_1_hour,precipitation_amount_next_6_hour,symbol_code_next_6_hour)
select [time],
[air_temperature],
wind_speed,
precipitation_amount_next_1_hour,
symbol_code_next_1_hour,
precipitation_amount_next_6_hour,
symbol_code_next_6_hour
from cteTemp;
现在的下一步是我已经创建了 table 具有多个坐标，我将在我的脚本任务中将其用作参数变量并添加一个 foreach 循环：
现在这将给我 3 行 JSON 对象，每个对象对应我的坐标 table 中存在的每组坐标，在 URL [=60] 中用作可变参数=]:
我现在遇到的问题是，当我将 JSON 对象插入我的 WeatherByHour table 时，它只使用第一行 JSON 对象的第一组坐标在我的例子中，它给了我 83 行，但理想情况下我希望它产生 83 + 83 + 83 = 249 行。这里最好的解决方案是什么，在脚本任务中创建一个大的 JSON 数组对象，或者以某种方式遍历 Weather table 中的所有 JSON 对象行并将它们插入 WeatherByHour table ?
这些是我的示例中使用的所有 table：
USE [Weather]
GO
/****** Object: Table [dbo].[Weather] Script Date: 2021-03-14 13:55:44 ******/
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
CREATE TABLE [dbo].[Weather](
[JSONData] [nvarchar](max) NOT NULL
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
USE [Weather]
GO
/****** Object: Table [dbo].[WeatherByHour] Script Date: 2021-03-14 13:56:09 ******/
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
CREATE TABLE [dbo].[WeatherByHour](
[id] [int] IDENTITY(1,1) NOT NULL,
[time] [datetime] NULL,
[air_temperature] [nvarchar](50) NULL,
[wind_speed] [nvarchar](50) NULL,
[precipitation_amount_next_1_hour] [nvarchar](50) NULL,
[symbol_code_next_1_hour] [nvarchar](50) NULL,
[precipitation_amount_next_6_hour] [nvarchar](50) NULL,
[symbol_code_next_6_hour] [nvarchar](50) NULL,
CONSTRAINT [PK_WeatherByHour] PRIMARY KEY CLUSTERED
(
[id] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
) ON [PRIMARY]
GO
create table Coordinates(
Municipality nvarchar(50),
Latitide nvarchar(50),
Longitude nvarchar(50)
)
INSERT INTO Coordinates (Municipality, Latitide, Longitude)
VALUES (114, 59.5166667, 17.9),
(115, 59.5833333, 18.2),
(117, 59.5, 18.45)
您可以使用交叉应用将时间序列转换为 json 行。
对于上面的示例 [Weather].[dbo].[Weather] table，您最终将得到 3（无坐标）x N（每个坐标的时间序列）。
SELECT s.value,
--JSON_VALUE(w.JSONData,'$.geometry.coordinates[1]') AS [latitude],
--JSON_VALUE(w.JSONData,'$.geometry.coordinates[0]') AS [longitude],
CAST(JSON_VALUE(s.value, '$.time') AS datetime) AS [time],
JSON_VALUE(s.value, '$.data.instant.details.air_temperature') as [air_temperature],
JSON_VALUE(s.value, '$.data.next_1_hours.details.precipitation_amount') as [precipitation_amount_next_1_hour]
FROM [Weather].[dbo].[Weather] w
CROSS APPLY OPENJSON(w.JSONData,'$.properties.timeseries') s