Dev/monamaki/use sql decimal

language-extensions/dotnet-core-CSharp/build/windows/build-dotnet-core-CSharp-extension.cmd

@@ -38,7 +38,7 @@ REM Do not call VsDevCmd if the environment is already set. Otherwise, it will k
 REM to the PATH environment variable and it will be too long for windows to handle.
 REM
 IF NOT DEFINED DevEnvDir (
-	CALL "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64
+	CALL "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 -host_arch=amd64
 )
 
 REM VSCMD_START_DIR set the working directory to this variable after calling VsDevCmd.bat
@@ -59,9 +59,9 @@ SET EXTENSION_HOST_INCLUDE=%ENL_ROOT%\extension-host\include
 SET DOTNET_NATIVE_LIB=%DOTNET_EXTENSION_HOME%\lib
 
 IF /I %BUILD_CONFIGURATION%==debug (
-	cl.exe /LD %DOTNET_NATIVE_SRC%\nativecsharpextension.cpp %DOTNET_NATIVE_SRC%\*.cpp /I %DOTNET_NATIVE_INCLUDE% /I %EXTENSION_HOST_INCLUDE% /D WINDOWS /D DEBUG /EHsc /Zi
+	cl.exe /LD %DOTNET_NATIVE_SRC%\nativecsharpextension.cpp %DOTNET_NATIVE_SRC%\*.cpp /I %DOTNET_NATIVE_INCLUDE% /I %EXTENSION_HOST_INCLUDE% /D WINDOWS /D DEBUG /EHsc /Zi /link /MACHINE:X64
 ) ELSE (
-	cl.exe /LD %DOTNET_NATIVE_SRC%\nativecsharpextension.cpp %DOTNET_NATIVE_SRC%\*.cpp /I %DOTNET_NATIVE_INCLUDE% /I %EXTENSION_HOST_INCLUDE% /D WINDOWS /EHsc /Zi
+	cl.exe /LD %DOTNET_NATIVE_SRC%\nativecsharpextension.cpp %DOTNET_NATIVE_SRC%\*.cpp /I %DOTNET_NATIVE_INCLUDE% /I %EXTENSION_HOST_INCLUDE% /D WINDOWS /EHsc /Zi /link /MACHINE:X64
 )
 
 CALL :CHECKERROR %ERRORLEVEL% "Error: Failed to build nativecsharpextension for configuration=%BUILD_CONFIGURATION%" || EXIT /b %ERRORLEVEL%

language-extensions/dotnet-core-CSharp/src/managed/CSharpInputDataSet.cs

@@ -9,8 +9,10 @@
 //
 //*********************************************************************
 using System;
+using System.Data.SqlTypes;
 using Microsoft.Data.Analysis;
 using static Microsoft.SqlServer.CSharpExtension.Sql;
+using static Microsoft.SqlServer.CSharpExtension.SqlNumericHelper;
 
 namespace Microsoft.SqlServer.CSharpExtension
 {
@@ -126,6 +128,9 @@ private unsafe void AddColumn(
                 case SqlDataType.DotNetReal:
                     AddDataFrameColumn<float>(columnNumber, rowsNumber, colData, colMap);
                     break;
+                case SqlDataType.DotNetNumeric:
+                    AddNumericDataFrameColumn(columnNumber, rowsNumber, colData, colMap);
+                    break;
                 case SqlDataType.DotNetChar:
                     int[] strLens = new int[rowsNumber];
                     Interop.Copy((int*)colMap, strLens, 0, (int)rowsNumber);
@@ -185,5 +190,53 @@ private unsafe void AddDataFrameColumn<T>(
 
             CSharpDataFrame.Columns.Add(colDataFrame);
         }
+
+        /// <summary>
+        /// This method adds NUMERIC/DECIMAL column data by converting from SQL_NUMERIC_STRUCT
+        /// to SqlDecimal values (full 38-digit precision), creating a PrimitiveDataFrameColumn<SqlDecimal>,
+        /// and adding it to the DataFrame.
+        /// 
+        /// IMPORTANT: We use SqlDecimal throughout to support SQL Server's full 38-digit precision.
+        /// C# decimal is NOT used to avoid 28-digit precision limitations and potential data loss.
+        /// </summary>
+        /// <param name="columnNumber">The column index.</param>
+        /// <param name="rowsNumber">Number of rows in this column.</param>
+        /// <param name="colData">Pointer to array of SQL_NUMERIC_STRUCT structures.</param>
+        /// <param name="colMap">Pointer to null indicator array (SQL_NULL_DATA for null values).</param>
+        private unsafe void AddNumericDataFrameColumn(
+            ushort columnNumber,
+            ulong  rowsNumber,
+            void   *colData,
+            int    *colMap)
+        {
+            // Cast the raw pointer to SQL_NUMERIC_STRUCT array
+            SqlNumericStruct* numericArray = (SqlNumericStruct*)colData;
+
+            // Create a DataFrame column for SqlDecimal values
+            // Using SqlDecimal instead of decimal provides full SQL Server precision (38 digits)
+            PrimitiveDataFrameColumn<SqlDecimal> colDataFrame = 
+                new PrimitiveDataFrameColumn<SqlDecimal>(_columns[columnNumber].Name, (int)rowsNumber);
+
+            // Convert each SQL_NUMERIC_STRUCT to SqlDecimal, handling nulls
+            Span<int> nullSpan = new Span<int>(colMap, (int)rowsNumber);
+            for (int i = 0; i < (int)rowsNumber; ++i)
+            {
+                // Check if this row has a null value
+                // 
+                // Why check both Nullable == 0 and SQL_NULL_DATA?
+                // - Nullable == 0 means column is declared NOT NULL (cannot contain nulls)
+                // - For NOT NULL columns, skip null checking for performance (nullSpan[i] is undefined)
+                // - For nullable columns (Nullable != 0), check if nullSpan[i] == SQL_NULL_DATA (-1)
+                // - This matches the pattern used by other numeric types in the codebase
+                if (_columns[columnNumber].Nullable == 0 || nullSpan[i] != SQL_NULL_DATA)
+                {
+                    // Convert SQL_NUMERIC_STRUCT to SqlDecimal with full precision support
+                    colDataFrame[i] = ToSqlDecimal(numericArray[i]);
+                }
+                // else: leave as null (default for nullable primitive column)
+            }
+
+            CSharpDataFrame.Columns.Add(colDataFrame);
+        }
     }
 }

language-extensions/dotnet-core-CSharp/src/managed/CSharpOutputDataSet.cs

@@ -9,12 +9,14 @@
 //
 //*********************************************************************
 using System;
+using System.Data.SqlTypes;
 using System.Linq;
 using System.Text;
 using System.Runtime.InteropServices;
 using System.Collections.Generic;
 using Microsoft.Data.Analysis;
 using static Microsoft.SqlServer.CSharpExtension.Sql;
+using static Microsoft.SqlServer.CSharpExtension.SqlNumericHelper;
 
 namespace Microsoft.SqlServer.CSharpExtension
 {
@@ -174,6 +176,9 @@ DataFrameColumn column
                 case SqlDataType.DotNetDouble:
                     SetDataPtrs<double>(columnNumber, GetArray<double>(column));
                     break;
+                case SqlDataType.DotNetNumeric:
+                    ExtractNumericColumn(columnNumber, column);
+                    break;
                 case SqlDataType.DotNetChar:
                     // Calculate column size from actual data.
                     // columnSize = max UTF-8 byte length across all rows.
@@ -203,7 +208,7 @@ DataFrameColumn column
         /// <summary>
         /// This method sets data pointer for the column and append the array to the handle list.
         /// </summary>
-        private unsafe void SetDataPtrs<T>(
+        private void SetDataPtrs<T>(
             ushort  columnNumber,
             T[]     array
         ) where T : unmanaged
@@ -213,6 +218,125 @@ T[]     array
             _handleList.Add(handle);
         }
 
+        /// <summary>
+        /// This method extracts NUMERIC/DECIMAL column data by converting SqlDecimal values
+        /// to SQL_NUMERIC_STRUCT array, pinning it, and storing the pointer.
+        /// 
+        /// Precision and Scale Terminology (T-SQL DECIMAL(precision, scale)):
+        /// - Precision: Total number of decimal digits (1-38), both left and right of decimal point
+        /// - Scale: Number of digits to the right of the decimal point (0-precision)
+        /// - Example: DECIMAL(10,2) can store values like 12345678.90 (10 total digits, 2 after decimal)
+        /// </summary>
+        /// <param name="columnNumber">The column index.</param>
+        /// <param name="column">The DataFrameColumn containing SqlDecimal values.</param>
+        private unsafe void ExtractNumericColumn(
+            ushort          columnNumber,
+            DataFrameColumn column)
+        {
+            if (column == null)
+            {
+                SetDataPtrs<SqlNumericStruct>(columnNumber, Array.Empty<SqlNumericStruct>());
+                return;
+            }
+
+            // Extract precision and scale from SqlDecimal values.
+            // SqlDecimal from Microsoft.Data.SqlClient preserves precision/scale metadata,
+            // so we find the maximum precision and scale across all non-null values.
+            //
+            // In T-SQL terms: We're determining the target DECIMAL(precision, scale)
+            // that can accommodate all values in this column.
+            //
+            byte precision = SqlNumericHelper.SQL_MIN_PRECISION;  // Start with minimum (1)
+            byte scale = (byte)_columns[columnNumber].DecimalDigits;
+
+            // Examine all rows to find maximum precision and scale
+            // This ensures we preserve the highest precision/scale present in the data
+            //
+            for (int rowNumber = 0; rowNumber < column.Length; ++rowNumber)
+            {
+                if (column[rowNumber] != null)
+                {
+                    SqlDecimal value = (SqlDecimal)column[rowNumber];
+
+                    // SqlDecimal already carries precision/scale metadata from the input
+                    // Use it directly - no need to recalculate
+                    if (!value.IsNull)
+                    {
+                        scale = Math.Max(scale, value.Scale);
+                        precision = Math.Max(precision, value.Precision);
+                    }
+                }
+            }
+
+            // Ensure precision is within T-SQL DECIMAL valid range (1-38)
+            precision = Math.Max(precision, SqlNumericHelper.SQL_MIN_PRECISION);
+            precision = Math.Min(precision, SqlNumericHelper.SQL_MAX_PRECISION);
+
+            // Ensure scale doesn't exceed precision (T-SQL DECIMAL(p,s) constraint: s <= p)
+            if (scale > precision)
+            {
+                precision = scale;
+            }
+
+            // Update column metadata with determined precision and scale
+            // IMPORTANT: For DECIMAL/NUMERIC types, Size represents precision (total digits),
+            // NOT byte size. This follows ODBC ColumnSize convention for SQL_NUMERIC/SQL_DECIMAL.
+            // DecimalDigits represents scale (digits after decimal point).
+            _columns[columnNumber].Size = precision;
+            _columns[columnNumber].DecimalDigits = scale;
+
+            Logging.Trace($"ExtractNumericColumn: Column {columnNumber}, T-SQL type=DECIMAL({precision},{scale}), RowCount={column.Length}");
+
+            // Convert each SqlDecimal value to SQL_NUMERIC_STRUCT
+            SqlNumericStruct[] numericArray = new SqlNumericStruct[column.Length];
+            for (int rowNumber = 0; rowNumber < column.Length; ++rowNumber)
+            {
+                if (column[rowNumber] != null)
+                {
+                    SqlDecimal value = (SqlDecimal)column[rowNumber];
+
+                    // Convert SqlDecimal to SQL_NUMERIC_STRUCT with target precision/scale
+                    // FromSqlDecimal handles scale adjustment if needed to match column metadata
+                    numericArray[rowNumber] = FromSqlDecimal(value, precision, scale);
+                    Logging.Trace($"ExtractNumericColumn: Row {rowNumber}, Value={value} converted to SqlNumericStruct");
+                }
+                else
+                {
+                    // For null values, create a zero-initialized struct with target precision/scale
+                    // The null indicator in strLenOrNullMap will mark this as SQL_NULL_DATA
+                    //
+                    // WHY create a struct for NULL values instead of leaving uninitialized?
+                    // - ODBC requires a valid struct pointer even for NULL values
+                    // - The strLenOrNullMap array separately tracks which values are NULL
+                    // - Native code may read from the struct pointer, so it must be valid memory
+                    // - We use sign=1 (positive) by convention for NULL placeholders
+                    SqlNumericStruct nullStruct = new SqlNumericStruct
+                    {
+                        precision = precision,
+                        scale = (sbyte)scale,
+                        sign = 1  // Positive sign convention for NULL placeholders
+                    };
+
+                    // Zero out the value array for NULL placeholders
+                    // Fixed buffer is already fixed - access directly via pointer
+                    unsafe
+                    {
+                        byte* valPtr = nullStruct.val;
+                        for (int i = 0; i < SqlNumericHelper.SQL_NUMERIC_VALUE_SIZE; i++)
+                        {
+                            valPtr[i] = 0;
+                        }
+                    }
+
+                    numericArray[rowNumber] = nullStruct;
+                    Logging.Trace($"ExtractNumericColumn: Row {rowNumber} is NULL");
+                }
+            }
+
+            // Pin the SqlNumericStruct array and store pointer
+            SetDataPtrs<SqlNumericStruct>(columnNumber, numericArray);
+        }
+
         /// <summary>
         /// This method gets the array from a DataFrameColumn Column for numeric types.
         /// </summary>