Interfacing with Python and C/C++ Libraries¶

This guide covers advanced techniques for integrating Cython with Python code, C libraries, C++ libraries, and managing the Global Interpreter Lock (GIL).

Python and Cython Integration¶

Function Visibility Levels¶

Cython provides three function declaration types:

# Python-only function (slow, full Python semantics)
def python_func(x):
    return x * 2

# C-only function (fast, no Python overhead)
cdef int c_func(int x):
    return x * 2

# Hybrid function (accessible from Python and C)
cpdef int hybrid_func(int x):
    return x * 2

Performance comparison: - cdef: Fastest, no Python overhead, C-only - cpdef: Fast from C, slight overhead from Python - def: Standard Python speed

Calling Conventions¶

# From Python
import mymodule
result = mymodule.hybrid_func(10)  # OK
# mymodule.c_func(10)  # ERROR: not accessible

# From Cython
cdef int result = c_func(10)  # Fast C call
result = hybrid_func(10)  # Also fast

Type Conversions¶

# Automatic Python ↔ C conversion
cpdef process_data(data):
    cdef int n = data  # Python → C conversion
    cdef double result = compute(n)
    return result  # C → Python conversion

# Manual conversions
from cpython cimport PyObject
cdef object py_obj = <object>c_pointer
cdef void* c_ptr = <void*>py_obj

Working with Python Objects¶

Efficient String Handling¶

from cpython.bytes cimport PyBytes_AsString, PyBytes_Size
from cpython.unicode cimport PyUnicode_AsUTF8String

def process_string(str text):
    """Convert Python string to C char*"""
    cdef bytes py_bytes = text.encode('utf-8')
    cdef char* c_string = PyBytes_AsString(py_bytes)
    cdef Py_ssize_t length = PyBytes_Size(py_bytes)

    # Work with C string
    cdef int i
    for i in range(length):
        # Process c_string[i]
        pass

    return text.upper()

List and Dict Operations¶

from cpython.list cimport PyList_New, PyList_SET_ITEM, PyList_GET_SIZE
from cpython.dict cimport PyDict_New, PyDict_SetItem, PyDict_GetItem

def fast_list_creation(int n):
    """Create list using C API"""
    cdef list result = PyList_New(n)
    cdef int i
    for i in range(n):
        PyList_SET_ITEM(result, i, i * i)
    return result

def fast_dict_ops():
    """Dictionary operations using C API"""
    cdef dict d = PyDict_New()
    cdef object key, value

    for i in range(1000):
        key = f"key_{i}"
        value = i * i
        PyDict_SetItem(d, key, value)

    return d

Reference Counting¶

from cpython.ref cimport Py_INCREF, Py_DECREF

cdef class ObjectHolder:
    cdef object held_object

    def __init__(self, obj):
        self.held_object = obj
        Py_INCREF(obj)  # Increase reference count

    def __dealloc__(self):
        Py_DECREF(self.held_object)  # Decrease on cleanup

Extension Types (cdef classes)¶

Basic Extension Type¶

cdef class Vector:
    """High-performance vector class"""
    cdef double x, y, z  # C-level attributes (fast, no dict overhead)
    cdef readonly int dimensions  # Read-only from Python

    def __init__(self, double x, double y, double z):
        self.x = x
        self.y = y
        self.z = z
        self.dimensions = 3

    cpdef double dot(self, Vector other):
        """Dot product"""
        return self.x * other.x + self.y * other.y + self.z * other.z

    cpdef Vector cross(self, Vector other):
        """Cross product"""
        return Vector(
            self.y * other.z - self.z * other.y,
            self.z * other.x - self.x * other.z,
            self.x * other.y - self.y * other.x
        )

    property magnitude:
        def __get__(self):
            return (self.x**2 + self.y**2 + self.z**2) ** 0.5

    def __repr__(self):
        return f"Vector({self.x}, {self.y}, {self.z})"

Inheritance and Composition¶

cdef class Shape:
    """Abstract base shape"""
    cdef double _area

    cdef double _compute_area(self):
        """C-only method for subclasses"""
        return 0.0

    cpdef double area(self):
        if self._area < 0:
            self._area = self._compute_area()
        return self._area

cdef class Circle(Shape):
    cdef double radius

    def __init__(self, double radius):
        self.radius = radius
        self._area = -1

    cdef double _compute_area(self):
        return 3.14159 * self.radius * self.radius

cdef class Rectangle(Shape):
    cdef double width, height

    def __init__(self, double width, double height):
        self.width = width
        self.height = height
        self._area = -1

    cdef double _compute_area(self):
        return self.width * self.height

Interfacing with C Libraries¶

Using Standard C Library¶

from libc.math cimport sin, cos, sqrt, pow, M_PI
from libc.stdlib cimport malloc, free, rand, srand, RAND_MAX
from libc.string cimport memcpy, memset
from libc.stdio cimport printf

def trigonometry(double angle):
    """Use C math functions"""
    cdef double rad = angle * M_PI / 180.0
    return {
        'sin': sin(rad),
        'cos': cos(rad),
        'tan': sin(rad) / cos(rad)
    }

def random_array(int n):
    """Generate random numbers using C rand()"""
    cdef int* arr = <int*>malloc(n * sizeof(int))
    cdef int i

    if arr is NULL:
        raise MemoryError()

    try:
        for i in range(n):
            arr[i] = rand() % 100

        # Convert to Python list
        return [arr[i] for i in range(n)]
    finally:
        free(arr)

External C Library Integration¶

Create mylib.h:

mylib.h

typedef struct {
    double x, y;
} Point2D;

double distance(Point2D* p1, Point2D* p2);
void initialize_library(void);

Create mylib.c:

mylib.c

#include "mylib.h"
#include <math.h>

double distance(Point2D* p1, Point2D* p2) {
    double dx = p1->x - p2->x;
    double dy = p1->y - p2->y;
    return sqrt(dx*dx + dy*dy);
}

void initialize_library(void) {
    // Setup code
}

Cython wrapper wrapper.pyx:

cdef extern from "mylib.h":
    ctypedef struct Point2D:
        double x
        double y

    double distance(Point2D* p1, Point2D* p2)
    void initialize_library()

# Initialize on import
initialize_library()

def calc_distance(double x1, double y1, double x2, double y2):
    """Python interface to C distance function"""
    cdef Point2D p1, p2
    p1.x = x1
    p1.y = y1
    p2.x = x2
    p2.y = y2
    return distance(&p1, &p2)

Update setup.py:

from setuptools import setup, Extension
from Cython.Build import cythonize

extensions = [
    Extension(
        "wrapper",
        sources=["wrapper.pyx", "mylib.c"],
        include_dirs=["."],
    )
]

setup(ext_modules=cythonize(extensions))

Working with Pointers and Arrays¶

from libc.stdlib cimport malloc, free

def manipulate_pointers():
    """Advanced pointer operations"""
    cdef int* int_ptr
    cdef double** matrix
    cdef int i, j

    # Allocate single pointer
    int_ptr = <int*>malloc(10 * sizeof(int))
    if int_ptr is NULL:
        raise MemoryError()

    try:
        for i in range(10):
            int_ptr[i] = i * i

        # Allocate 2D array
        matrix = <double**>malloc(5 * sizeof(double*))
        if matrix is NULL:
            raise MemoryError()

        for i in range(5):
            matrix[i] = <double*>malloc(5 * sizeof(double))
            if matrix[i] is NULL:
                raise MemoryError()

            for j in range(5):
                matrix[i][j] = i * j

        # Use data...
        result = [[matrix[i][j] for j in range(5)] for i in range(5)]

        # Free 2D array
        for i in range(5):
            free(matrix[i])
        free(matrix)

        return result
    finally:
        free(int_ptr)

C++ Integration¶

Basic C++ Class Wrapping¶

C++ header shape.hpp:

// shape.hpp
#include <string>

class Shape {
public:
    Shape(double x, double y) : x_(x), y_(y) {}
    virtual ~Shape() {}

    double getX() const { return x_; }
    double getY() const { return y_; }
    virtual double area() const = 0;
    virtual std::string name() const = 0;

protected:
    double x_, y_;
};

class Circle : public Shape {
public:
    Circle(double x, double y, double radius) 
        : Shape(x, y), radius_(radius) {}

    double area() const override {
        return 3.14159 * radius_ * radius_;
    }

    std::string name() const override {
        return "Circle";
    }

private:
    double radius_;
};

Cython wrapper shapes.pyx:

# distutils: language = c++

from libcpp.string cimport string

cdef extern from "shape.hpp":
    cdef cppclass Shape:
        Shape(double x, double y)
        double getX()
        double getY()
        double area()
        string name()

    cdef cppclass Circle(Shape):
        Circle(double x, double y, double radius)

cdef class PyShape:
    """Python wrapper for C++ Shape"""
    cdef Shape* c_shape

    def __cinit__(self):
        self.c_shape = NULL

    def __dealloc__(self):
        if self.c_shape != NULL:
            del self.c_shape

    @property
    def x(self):
        return self.c_shape.getX()

    @property
    def y(self):
        return self.c_shape.getY()

    def area(self):
        return self.c_shape.area()

    def name(self):
        return self.c_shape.name().decode('utf-8')

cdef class PyCircle(PyShape):
    """Python wrapper for C++ Circle"""
    def __cinit__(self, double x, double y, double radius):
        self.c_shape = new Circle(x, y, radius)

STL Containers¶

# distutils: language = c++

from libcpp.vector cimport vector
from libcpp.map cimport map as cpp_map
from libcpp.set cimport set as cpp_set
from libcpp.string cimport string
from libcpp.pair cimport pair

def use_stl_containers():
    """Demonstrate C++ STL usage"""
    cdef vector[int] vec
    cdef cpp_map[string, int] word_count
    cdef cpp_set[int] unique_nums

    # Vector operations
    vec.push_back(1)
    vec.push_back(2)
    vec.push_back(3)

    # Map operations
    word_count[b"hello"] = 5
    word_count[b"world"] = 3

    # Set operations
    unique_nums.insert(1)
    unique_nums.insert(2)
    unique_nums.insert(2)  # Duplicate ignored

    # Convert to Python
    py_vec = [vec[i] for i in range(vec.size())]
    py_map = {k.decode('utf-8'): v for k, v in word_count}
    py_set = {x for x in unique_nums}

    return py_vec, py_map, py_set

GIL Management¶

Releasing the GIL¶

from cython.parallel import prange, parallel
import numpy as np

def compute_intensive(double[:] data):
    """Release GIL for CPU-bound operations"""
    cdef Py_ssize_t i, n = data.shape[0]
    cdef double result = 0.0

    # Release GIL - no Python API calls allowed
    with nogil:
        for i in range(n):
            result += data[i] * data[i]

    return result

cdef double _nogil_function(double x) nogil:
    """Pure C function that never needs GIL"""
    return x * x + 2 * x + 1

def parallel_compute(double[:] data):
    """Parallel computation without GIL"""
    cdef Py_ssize_t i, n = data.shape[0]
    cdef double[:] result = np.empty(n)

    with nogil, parallel():
        for i in prange(n):
            result[i] = _nogil_function(data[i])

    return np.asarray(result)

GIL Acquisition¶

cdef void callback_with_python(void* user_data) nogil:
    """Call Python from C callback"""
    with gil:
        # Now we can use Python API
        py_obj = <object>user_data
        py_obj.some_method()

cdef extern from "external.h":
    void register_callback(void (*callback)(void*) nogil)

def setup_callback(obj):
    """Register callback that needs Python"""
    cdef void* ptr = <void*>obj
    register_callback(callback_with_python)

Thread Safety¶

from cpython.ref cimport PyObject
from threading import Lock

cdef class ThreadSafeCounter:
    cdef int count
    cdef object lock  # Python lock object

    def __init__(self):
        self.count = 0
        self.lock = Lock()

    def increment(self):
        with self.lock:
            self.count += 1

    def get(self):
        with self.lock:
            return self.count

    cpdef int fast_increment(self):
        """C-level increment (not thread-safe!)"""
        self.count += 1
        return self.count

Callbacks and Function Pointers¶

# Callback from C to Python
ctypedef void (*c_callback)(double result, void* user_data)

cdef void call_python_callback(double result, void* user_data) with gil:
    """C callback that calls Python"""
    callback = <object>user_data
    callback(result)

def register_callback(callback_func):
    """Register Python callback"""
    cdef void* user_data = <void*>callback_func
    # Register with C library
    # c_library_register(call_python_callback, user_data)

Memory Management Best Practices¶

from libc.stdlib cimport malloc, calloc, realloc, free
from cpython.mem cimport PyMem_Malloc, PyMem_Free

cdef class ManagedArray:
    """RAII-style memory management"""
    cdef double* data
    cdef Py_ssize_t size

    def __cinit__(self, Py_ssize_t size):
        self.size = size
        self.data = <double*>PyMem_Malloc(size * sizeof(double))
        if self.data is NULL:
            raise MemoryError()

    def __dealloc__(self):
        if self.data is not NULL:
            PyMem_Free(self.data)

    def __getitem__(self, Py_ssize_t i):
        if i < 0 or i >= self.size:
            raise IndexError()
        return self.data[i]

    def __setitem__(self, Py_ssize_t i, double value):
        if i < 0 or i >= self.size:
            raise IndexError()
        self.data[i] = value