feat: update to work with openssl 3.x

Author: stakach

Gemfile

@@ -1,4 +1,2 @@
 source 'https://rubygems.org'
 gemspec
-
-gem "rubysl", :platform => :rbx

lib/ruby-tls/ssl.rb

@@ -34,9 +34,9 @@ module SSL
 
         SSL_ST_OK = 0x03
         begin
-            attach_function :SSL_library_init, [], :int
             attach_function :SSL_load_error_strings, [], :void
             attach_function :ERR_load_crypto_strings, [], :void
+            attach_function :SSL_library_init, [], :int
 
             attach_function :SSL_state, [:ssl], :int
             def self.SSL_is_init_finished(ssl)
@@ -76,7 +76,9 @@ def self.SSL_is_init_finished(ssl)
         attach_function :BIO_free, [:bio], :int
 
         # GetPeerCert
-        attach_function :SSL_get_peer_certificate, [:ssl], :x509
+        attach_function :SSL_get1_peer_certificate, [:ssl], :x509
+
+        attach_function :SSL_CTX_set_security_level, [:ssl, :int], :void
 
 
         # PutPlaintext
@@ -207,6 +209,8 @@ def self.SSL_CTX_set_tlsext_servername_callback(ctx, callback)
         attach_function :SSL_CTX_use_PrivateKey_file, [:ssl_ctx, :string, :int], :int, :blocking => true
         attach_function :SSL_CTX_use_PrivateKey, [:ssl_ctx, :pointer], :int
         attach_function :ERR_print_errors_fp, [:pointer], :void     # Pointer == File Handle
+        attach_function :ERR_get_error, [], :long
+        attach_function :ERR_reason_error_string, [:ulong], :string
         attach_function :SSL_CTX_use_certificate_chain_file, [:ssl_ctx, :string], :int, :blocking => true
         attach_function :SSL_CTX_use_certificate, [:ssl_ctx, :x509], :int
         attach_function :SSL_CTX_set_cipher_list, [:ssl_ctx, :string], :int
@@ -310,13 +314,18 @@ def self.SSL_CTX_set_tlsext_servername_callback(ctx, callback)
         @init_required ||= false
         unless @init_required
             if OPENSSL_V1_1
-                self.OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, ::FFI::Pointer::NULL)
+                # self.OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, ::FFI::Pointer::NULL)
             else
-                self.SSL_load_error_strings
-                self.SSL_library_init
-                self.ERR_load_crypto_strings
+                # self.SSL_load_error_strings
+                # self.SSL_library_init
+                # self.ERR_load_crypto_strings
             end
 
+            # self.SSL_load_error_strings
+            # self.SSL_library_init
+            # self.ERR_load_crypto_strings
+            self.OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, ::FFI::Pointer::NULL)
+
             # Setup multi-threaded support
             #SSL_LOCKS = []
             #num_locks = self.CRYPTO_num_locks
@@ -325,7 +334,6 @@ def self.SSL_CTX_set_tlsext_servername_callback(ctx, callback)
             #self.CRYPTO_set_locking_callback(LockingCB)
             #self.CRYPTO_set_id_callback(ThreadIdCB)
 
-
             bio = self.BIO_new_mem_buf(PrivateMaterials, PrivateMaterials.bytesize)
 
             # Get the private key structure
@@ -384,11 +392,13 @@ def initialize(server, options = {})
 
                 if @is_server
                     @ssl_ctx = SSL.SSL_CTX_new(SSL.TLS_server_method)
+                    SSL.SSL_CTX_set_security_level(@ssl_ctx, 0)
                     set_private_key(options[:private_key] || SSL::DEFAULT_PRIVATE)
                     set_certificate(options[:cert_chain]  || SSL::DEFAULT_CERT)
                     set_client_ca(options[:client_ca])
                 else
                     @ssl_ctx = SSL.SSL_CTX_new(SSL.TLS_client_method)
+                    SSL.SSL_CTX_set_security_level(@ssl_ctx, 0)
                 end
 
                 SSL.SSL_CTX_set_options(@ssl_ctx, SSL::SSL_OP_ALL)
@@ -517,8 +527,10 @@ def set_certificate(cert)
                 end
 
                 if err <= 0
+                    error_code = SSL.ERR_get_error
+                    reason = SSL.ERR_reason_error_string(error_code)
                     cleanup
-                    raise 'invalid certificate or file not found'
+                    raise "invalid certificate or file not found: #{err} #{reason}"
                 end
             end
 
@@ -616,7 +628,7 @@ def remove_host(host_name)
 
             def get_peer_cert
                 return '' unless @ready
-                SSL.SSL_get_peer_certificate(@ssl)
+                SSL.SSL_get1_peer_certificate(@ssl)
             end
 
             def negotiated_protocol
@@ -657,7 +669,7 @@ def encrypt(data)
             def decrypt(data)
                 return unless @ready
 
-                put_cipher_text data
+                result = put_cipher_text data
 
                 if not SSL.SSL_is_init_finished(@ssl)
                     resp = @is_server ? SSL.SSL_accept(@ssl) : SSL.SSL_connect(@ssl)
@@ -754,7 +766,8 @@ def get_plain_text(buffer, ready)
                 if size >= 0
                     size
                 else
-                    SSL.SSL_get_error(@ssl, size) == SSL_ERROR_WANT_READ ? 0 : -1
+                    error_code = SSL.SSL_get_error(@ssl, size)
+                    error_code == SSL_ERROR_WANT_READ ? 0 : -1
                 end
             end
 
@@ -795,7 +808,7 @@ def put_cipher_text(data)
             SSL_ERROR_WANT_WRITE = 3
             def put_plain_text(data)
                 @write_queue.push(data) if data
-                return 0 unless SSL.SSL_is_init_finished(@ssl)
+                # return 0 unless SSL.SSL_is_init_finished(@ssl)
 
                 fatal = false
                 did_work = false

lib/ruby-tls/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubyTls
-    VERSION = '2.4.0'
+    VERSION = '3.0.0'
 end

spec/comms_spec.rb

@@ -101,19 +101,19 @@ def handshake_cb(protocol)
             @server = Server1.new(@client, @server_data, @interleaved)
             @client.server = @server
 
-
             @client.ssl.start
             @client.ssl.cleanup
             @server.ssl.cleanup
 
-
             # Calls to encrypt should not cause crashes after cleanup
             @server.ssl.encrypt('server response')
             @client.ssl.encrypt('client request')
 
+            expect(@server.stop).to eq(nil)
+
+            expect(@interleaved).to eq(['server ready', 'client ready', 'client request', 'server response'])
             expect(@server_data).to eq(['ready', 'client request'])
             expect(@client_data).to eq(['ready', 'server response'])
-            expect(@interleaved).to eq(['server ready', 'client ready', 'client request', 'server response'])
         end
     end
 end