1 /++ 2 Utilities for $(LINK2 https://docs.python.org/3/c-api/buffer.html, Python Buffer Protocol). 3 4 License: $(HTTP www.apache.org/licenses/LICENSE-2.0, Apache-2.0) 5 Copyright: 2020 Ilya Yaroshenko, Kaleidic Associates Advisory Limited, Symmetry Investments 6 Authors: Ilya Yaroshenko 7 8 Macros: 9 SUBREF = $(REF_ALTTEXT $(TT $2), $2, mir, ndslice, $1)$(NBSP) 10 T2=$(TR $(TDNW $(LREF $1)) $(TD $+)) 11 T4=$(TR $(TDNW $(LREF $1)) $(TD $2) $(TD $3) $(TD $4)) 12 STD = $(TD $(SMALL $0)) 13 PGB = $(LINK2 https://docs.python.org/3/c-api/buffer.html#c.PyObject_GetBuffer, PyObject_GetBuffer()) 14 +/ 15 module mir.ndslice.connect.cpython; 16 17 import mir.ndslice.slice; 18 import core.stdc.config; 19 import std.traits; 20 21 /++ 22 Construct flags for $(PGB). 23 If `T` is not `const` or `immutable` then the flags require writable buffer. 24 If slice kind is $(SUBREF slice, Contiguous) then the flags require $(LINK2 https://docs.python.org/3/c-api/buffer.html#contiguity-requests, c_contiguous) buffer. 25 26 Params: 27 kind = slice kind 28 T = record type 29 Returns: 30 flags for $(LREF Py_buffer) request. 31 +/ 32 enum int pythonBufferFlags(SliceKind kind, T) = (kind == Contiguous ? PyBuf_c_contiguous : PyBuf_strides) | (is(T == const) || is (T == immutable) ? PyBuf_records_ro : PyBuf_records); 33 34 /++ 35 Fills the slice (structure) from the python `view`. 36 The view should be created by $(PGB) that was called with $(LREF pythonBufferFlags). 37 38 Params: 39 slice = output ndslice 40 view = $(LREF Py_buffer) requested 41 Returns: 42 one of the `input_buffer_*` $(LREF PythonBufferErrorCode) on failure and `success` otherwise. 43 +/ 44 PythonBufferErrorCode fromPythonBuffer(T, size_t N, SliceKind kind)(ref Slice!(T*, N, kind) slice, ref const Py_buffer view) nothrow @nogc @trusted 45 if (N <= PyBuf_max_ndim) 46 { 47 import core.stdc..string: strcmp; 48 import mir.internal.utility: Iota; 49 50 static if (!(is(T == const) || is(T == immutable))) 51 assert(!view.readonly); 52 53 enum N = slice.N; 54 enum S = slice.S; 55 56 if (N != view.ndim) 57 return typeof(return).input_buffer_ndim_mismatch; 58 if (T.sizeof != view.itemsize) 59 return typeof(return).input_buffer_itemsize_mismatch; 60 if (pythonBufferFormat!(Unqual!T).ptr.strcmp(view.format)) 61 return typeof(return).input_buffer_format_mismatch; 62 if (kind == Canonical && view.strides[N - 1] != T.sizeof) 63 return typeof(return).input_buffer_strides_mismatch; 64 65 foreach(i; Iota!N) 66 slice._lengths[i] = view.shape[i]; 67 foreach(i; Iota!S) 68 { 69 assert(view.strides[i] % T.sizeof == 0); 70 slice._strides[i] = view.strides[i] / T.sizeof; 71 } 72 slice._iterator = cast(T*) view.buf; 73 74 return typeof(return).success; 75 } 76 77 /// 78 version(mir_test) unittest 79 { 80 import mir.ndslice.slice: Slice; 81 auto bar(ref const Py_buffer view) 82 { 83 Slice!(const(double)*, 2) mat; 84 if (auto error = mat.fromPythonBuffer(view)) 85 { 86 // has null pointer 87 } 88 return mat; 89 } 90 } 91 92 /++ 93 Fills the python view (structure) from the slice. 94 Params: 95 slice = input ndslice 96 view = output $(LREF Py_buffer). 97 $(LREF Py_buffer.internal) is initialized with null value, 98 $(LREF Py_buffer.obj) is not initialized. 99 Other $(LREF Py_buffer) fields are initialized according to the flags and slice. 100 flags = requester flags 101 structureBuffer = Single chunk of memory with the same alignment and size as $(SUBREF _slice, Structure). 102 The buffer is used to store shape and strides for the view. 103 Returns: 104 one of the `cannot_create_*` $(LREF PythonBufferErrorCode) on failure and `success` otherwise. 105 +/ 106 PythonBufferErrorCode toPythonBuffer(T, size_t N, SliceKind kind)(Slice!(T*, N, kind) slice, ref Py_buffer view, int flags, ref Structure!N structureBuffer) nothrow @nogc @trusted 107 if (N <= PyBuf_max_ndim) 108 { 109 structureBuffer.lengths = slice._lengths; 110 structureBuffer.strides = slice.strides; 111 112 foreach(ref stride; structureBuffer.strides) 113 stride *= T.sizeof; 114 115 ///////////////////// 116 /// always filled /// 117 ///////////////////// 118 view.buf = slice._iterator; 119 // skip view.obj 120 view.len = slice.elementCount * T.sizeof; 121 view.itemsize = T.sizeof; 122 view.ndim = N; 123 view.internal = null; 124 125 static if (kind != Contiguous) 126 { 127 bool check_single_memory_block; 128 } 129 130 /// shape /// 131 if ((flags & PyBuf_nd) == PyBuf_nd) 132 { 133 view.shape = cast(sizediff_t*) structureBuffer.lengths.ptr; 134 /// strides /// 135 if ((flags & PyBuf_strides) == PyBuf_strides) 136 view.strides = cast(sizediff_t*) structureBuffer.strides.ptr; 137 else 138 { 139 view.strides = null; 140 static if (kind != Contiguous) 141 check_single_memory_block = true; 142 } 143 } 144 else 145 { 146 view.shape = null; 147 view.strides = null; 148 static if (kind != Contiguous) 149 check_single_memory_block = true; 150 } 151 view.suboffsets = null; 152 153 /// ! structure verification ! /// 154 static if (kind == Contiguous) 155 { 156 static if (N != 1) 157 { 158 if ((flags & PyBuf_f_contiguous) == PyBuf_f_contiguous) 159 { 160 import mir.ndslice.dynamic: everted; 161 import mir.ndslice.topology: iota; 162 if (slice.everted.shape.iota.everted.strides != slice.strides) 163 return typeof(return).cannot_create_f_contiguous_buffer; 164 } 165 } 166 } 167 else 168 { 169 import mir.ndslice.dynamic: everted, normalizeStructure; 170 import mir.ndslice.topology: iota; 171 if ((flags & PyBuf_c_contiguous) == PyBuf_c_contiguous) 172 { 173 if (slice.shape.iota.strides != slice.strides && slice.everted.shape.iota.everted.strides != slice.strides) 174 return typeof(return).cannot_create_c_contiguous_buffer; 175 } 176 else 177 if ((flags & PyBuf_any_contiguous) == PyBuf_any_contiguous) 178 { 179 if (slice.shape.iota.strides != slice.strides && slice.everted.shape.iota.everted.strides != slice.strides) 180 return typeof(return).cannot_create_any_contiguous_buffer; 181 } 182 else 183 if (check_single_memory_block) 184 { 185 if (!slice.normalizeStructure) 186 return typeof(return).cannot_create_a_buffer_without_strides; 187 } 188 } 189 190 /// readonly /// 191 static if (is(T == const) || is(T == immutable)) 192 { 193 if (flags & PyBuf_writable) 194 return typeof(return).cannot_create_writable_buffer; 195 view.readonly = 1; 196 } 197 else 198 view.readonly = 0; 199 200 /// format /// 201 if (flags & PyBuf_format) 202 { 203 enum fmt = pythonBufferFormat!(Unqual!T); 204 static if (fmt is null) 205 return typeof(return).cannot_create_format_string; 206 else 207 view.format = cast(char*)fmt.ptr; 208 } 209 else 210 view.format = null; 211 212 return typeof(return).success; 213 } 214 215 /// 216 version(mir_test) unittest 217 { 218 import mir.ndslice.slice : Slice, Structure, Universal, Contiguous, SliceKind; 219 Py_buffer bar(SliceKind kind)(Slice!(double*, 2, kind) slice) 220 { 221 import core.stdc.stdlib; 222 enum N = 2; 223 224 auto structurePtr = cast(Structure!N*) Structure!N.sizeof.malloc; 225 if (!structurePtr) 226 assert(0); 227 Py_buffer view; 228 229 if (auto error = slice.toPythonBuffer(view, PyBuf_records_ro, *structurePtr)) 230 { 231 view = view.init; // null buffer 232 structurePtr.free; 233 } 234 else 235 { 236 assert(cast(sizediff_t*)&structurePtr.lengths == view.shape); 237 assert(cast(sizediff_t*)&structurePtr.strides == view.strides); 238 } 239 240 return view; 241 } 242 243 alias barUni = bar!Universal; 244 alias barCon = bar!Contiguous; 245 } 246 247 /// Python $(LINK2 https://docs.python.org/3/c-api/buffer.html#buffer-structure, Buffer structure). 248 extern(C) 249 struct bufferinfo 250 { 251 /// 252 void *buf; 253 /// 254 void *obj; 255 /// 256 sizediff_t len; 257 /// 258 sizediff_t itemsize; 259 /// 260 int readonly; 261 /// 262 int ndim; 263 /// 264 char *format; 265 /// 266 sizediff_t *shape; 267 /// 268 sizediff_t *strides; 269 /// 270 sizediff_t *suboffsets; 271 /// 272 void *internal; 273 } 274 /// ditto 275 alias Py_buffer = bufferinfo; 276 277 /++ 278 Error codes for ndslice - Py_buffer conversion. 279 +/ 280 enum PythonBufferErrorCode 281 { 282 /// 283 success, 284 /// 285 cannot_create_format_string, 286 /// 287 cannot_create_writable_buffer, 288 /// 289 cannot_create_f_contiguous_buffer, 290 /// 291 cannot_create_c_contiguous_buffer, 292 /// 293 cannot_create_any_contiguous_buffer, 294 /// 295 cannot_create_a_buffer_without_strides, 296 /// 297 input_buffer_ndim_mismatch, 298 /// 299 input_buffer_itemsize_mismatch, 300 /// 301 input_buffer_format_mismatch, 302 /// 303 input_buffer_strides_mismatch, 304 } 305 306 /// 307 enum PyBuf_max_ndim = 64; 308 309 /// 310 enum PyBuf_simple = 0; 311 /// 312 enum PyBuf_writable = 0x0001; 313 /// 314 enum PyBuf_writeable = PyBuf_writable; 315 /// 316 enum PyBuf_format = 0x0004; 317 /// 318 enum PyBuf_nd = 0x0008; 319 /// 320 enum PyBuf_strides = (0x0010 | PyBuf_nd); 321 /// 322 enum PyBuf_c_contiguous = (0x0020 | PyBuf_strides); 323 /// 324 enum PyBuf_f_contiguous = (0x0040 | PyBuf_strides); 325 /// 326 enum PyBuf_any_contiguous = (0x0080 | PyBuf_strides); 327 /// 328 enum PyBuf_indirect = (0x0100 | PyBuf_strides); 329 330 /// 331 enum PyBuf_contig = (PyBuf_nd | PyBuf_writable); 332 /// 333 enum PyBuf_contig_ro = (PyBuf_nd); 334 335 /// 336 enum PyBuf_strided = (PyBuf_strides | PyBuf_writable); 337 /// 338 enum PyBuf_strided_ro = (PyBuf_strides); 339 340 /// 341 enum PyBuf_records = (PyBuf_strides | PyBuf_writable | PyBuf_format); 342 /// 343 enum PyBuf_records_ro = (PyBuf_strides | PyBuf_format); 344 345 /++ 346 Returns $(HTTPS docs.python.org/3/c-api/buffer.html#c.Py_buffer.format, python format (type)) string. 347 For example, `"O"` for `PyObject` and "B" for ubyte. 348 +/ 349 template pythonBufferFormat(T) 350 { 351 static if (is(T == struct) && __traits(identifier, A) == "PyObject") 352 enum pythonBufferFormat = "O"; 353 else 354 static if (is(Unqual!T == short)) 355 enum pythonBufferFormat = "h"; 356 else 357 static if (is(Unqual!T == ushort)) 358 enum pythonBufferFormat = "H"; 359 else 360 static if (is(Unqual!T == int)) 361 enum pythonBufferFormat = "i"; 362 else 363 static if (is(Unqual!T == uint)) 364 enum pythonBufferFormat = "I"; 365 else 366 static if (is(Unqual!T == float)) 367 enum pythonBufferFormat = "f"; 368 else 369 static if (is(Unqual!T == double)) 370 enum pythonBufferFormat = "d"; 371 else 372 static if (is(Unqual!T == long)) 373 enum pythonBufferFormat = "q"; 374 else 375 static if (is(Unqual!T == ulong)) 376 enum pythonBufferFormat = "Q"; 377 else 378 static if (is(Unqual!T == ubyte)) 379 enum pythonBufferFormat = "B"; 380 else 381 static if (is(Unqual!T == byte)) 382 enum pythonBufferFormat = "b"; 383 else 384 static if (is(Unqual!T == char)) 385 enum pythonBufferFormat = "c"; 386 else 387 static if (is(Unqual!T == char*)) 388 enum pythonBufferFormat = "z"; 389 else 390 static if (is(Unqual!T == void*)) 391 enum pythonBufferFormat = "P"; 392 else 393 static if (is(Unqual!T == bool)) 394 enum pythonBufferFormat = "?"; 395 else 396 static if (is(Unqual!T == wchar*)) 397 enum pythonBufferFormat = "Z"; 398 else 399 static if (is(Unqual!T == wchar)) 400 enum pythonBufferFormat = "u"; 401 else 402 { 403 static if (is(cpp_long)) 404 { 405 static if (is(Unqual!T == cpp_long)) 406 enum pythonBufferFormat = "l"; 407 else 408 enum pythonBufferFormat = null; 409 } 410 else 411 static if (is(cpp_ulong)) 412 { 413 static if (is(Unqual!T == cpp_ulong)) 414 enum pythonBufferFormat = "L"; 415 else 416 enum pythonBufferFormat = null; 417 } 418 else 419 static if (is(c_long_double)) 420 { 421 static if (is(Unqual!T == c_long_double)) 422 enum pythonBufferFormat = "g"; 423 else 424 enum pythonBufferFormat = null; 425 } 426 else 427 enum pythonBufferFormat = null; 428 } 429 }