fix: Add critical memory safety and error handling improvements

- Add NULL checks after malloc() calls in HTTP/HTTPS thread creation
- Add error checking for fcntl() calls to prevent silent failures
- Add integer overflow protection in WebSocket frame handling
- Improve socket option error handling with proper validation
- Add SIZE_MAX check in ws_send_frame to prevent overflow

These fixes address potential crashes and security issues in high-load
scenarios and improve overall robustness of the server.

src/server.c

@@ -208,7 +208,15 @@ void configure_ssl_context(SSL_CTX *ctx)
 void set_socket_options(int socket_fd)
 {
     int flags = fcntl(socket_fd, F_GETFL, 0);
-    fcntl(socket_fd, F_SETFL, flags | O_NONBLOCK); // Make socket non-blocking
+    if (flags == -1)
+    {
+        perror("fcntl F_GETFL");
+        return;
+    }
+    if (fcntl(socket_fd, F_SETFL, flags | O_NONBLOCK) == -1)
+    {
+        perror("fcntl F_SETFL");
+    }
 
     int reuse = 1;
     int keepalive = 1;
@@ -318,6 +326,13 @@ void *start_http_server(void *arg)
                 {
                     pthread_t client_thread;
                     int *client_socket_ptr = malloc(sizeof(int));
+                    if (!client_socket_ptr)
+                    {
+                        perror("Failed to allocate memory for client socket");
+                        close(client_socket);
+                        pthread_mutex_unlock(&thread_count_mutex);
+                        continue;
+                    }
                     *client_socket_ptr = client_socket;
 
                     if (pthread_create(&client_thread, NULL, handle_http_client, client_socket_ptr) == 0)
@@ -401,6 +416,13 @@ void *start_https_server(void *arg)
         {
             pthread_t client_thread;
             int *client_socket_ptr = malloc(sizeof(int));
+            if (!client_socket_ptr)
+            {
+                perror("Failed to allocate memory for client socket");
+                close(client_socket);
+                pthread_mutex_unlock(&thread_count_mutex);
+                continue;
+            }
             *client_socket_ptr = client_socket;
 
             if (pthread_create(&client_thread, NULL, handle_https_client, client_socket_ptr) == 0)
@@ -826,7 +848,10 @@ void *handle_https_client(void *arg)
 
             // Set socket to non-blocking mode for HTTP/2
             int flags = fcntl(client_socket, F_GETFL, 0);
-            fcntl(client_socket, F_SETFL, flags | O_NONBLOCK);
+            if (flags != -1)
+            {
+                fcntl(client_socket, F_SETFL, flags | O_NONBLOCK);
+            }
 
             // Initialize HTTP/2 session
             http2_session_t h2_session;

src/websocket.c

@@ -2,6 +2,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <stdint.h>
 #include <unistd.h>
 #include <openssl/sha.h>
 #include <openssl/bio.h>
@@ -242,6 +243,12 @@ int ws_create_frame(uint8_t *buffer, size_t buffer_size, uint8_t opcode, const u
 int ws_send_frame(ws_connection_t *conn, uint8_t opcode, const uint8_t *payload, size_t payload_len)
 {
     // Allocate buffer with enough space for header (max 10 bytes) + payload
+    // Check for integer overflow
+    if (payload_len > SIZE_MAX - 10)
+    {
+        return -1;
+    }
+    
     size_t max_frame_size = 10 + payload_len;
     if (max_frame_size > 65536)
     {