Merge branch 'master' of https://github.com/larskanis/pg-ldap-sync

.github/workflows/ci.yml

@@ -0,0 +1,81 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  job_test_gem:
+    name: Test built gem
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: windows
+            ruby: "head"
+            PGVERSION: 15.1-1-windows-x64
+            PGVER: "15"
+          - os: windows
+            ruby: "2.4"
+            PGVERSION: 9.4.26-1-windows-x64
+            PGVER: "9.4"
+          - os: ubuntu
+            ruby: "head"
+            PGVER: "15"
+          - os: ubuntu
+            os_ver: "20.04"
+            ruby: "2.3"
+            PGVER: "9.3"
+          - os: macos
+            ruby: "head"
+            PGVERSION: 15.1-1-osx
+            PGVER: "15"
+
+    runs-on: ${{ matrix.os }}-${{ matrix.os_ver || 'latest' }}
+    env:
+      PGVERSION: ${{ matrix.PGVERSION }}
+      PGVER: ${{ matrix.PGVER }}
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+
+      - name: Download PostgreSQL Windows
+        if: matrix.os == 'windows'
+        run: |
+          Add-Type -AssemblyName System.IO.Compression.FileSystem
+          function Unzip {
+              param([string]$zipfile, [string]$outpath)
+              [System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath)
+          }
+
+          $(new-object net.webclient).DownloadFile("http://get.enterprisedb.com/postgresql/postgresql-$env:PGVERSION-binaries.zip", "postgresql-binaries.zip")
+          Unzip "postgresql-binaries.zip" "."
+          echo "$pwd/pgsql/bin"  | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "PGUSER=$env:USERNAME"  | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          echo "PGPASSWORD="  | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          md temp
+          icacls temp /grant "Everyone:(OI)(CI)F" /T
+
+      - name: Download PostgreSQL Ubuntu
+        if: matrix.os == 'ubuntu'
+        run: |
+          echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main $PGVER" | sudo tee -a /etc/apt/sources.list.d/pgdg.list
+          wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+          sudo apt-get -y update
+          sudo apt-get -y --allow-downgrades install postgresql-$PGVER libpq5=$PGVER* libpq-dev=$PGVER*
+          echo /usr/lib/postgresql/$PGVER/bin >> $GITHUB_PATH
+
+      - name: Download PostgreSQL Macos
+        if: matrix.os == 'macos'
+        run: |
+          wget https://get.enterprisedb.com/postgresql/postgresql-$PGVERSION-binaries.zip && \
+          sudo mkdir -p /Library/PostgreSQL && \
+          sudo unzip postgresql-$PGVERSION-binaries.zip -d /Library/PostgreSQL/$PGVER && \
+          echo /Library/PostgreSQL/$PGVER/bin >> $GITHUB_PATH
+
+      - run: bundle install
+
+      - name: Run specs
+        run: bundle exec rake test

CHANGELOG.md

@@ -1,3 +1,17 @@
+## 0.5.0 / 2023-02-03
+
+* Add Kerberos and NTLM authentication support
+
+
+## 0.4.0 / 2022-12-02
+
+* Support groups with over 1500 users in Active Directory server. #32
+* Retrieve only necessary attributes from LDAP server.
+* Add error text to exception, so that it's visible even if nothing is logged.
+* Fix compatibility with PostgreSQL-15
+* Require ruby-2.3+
+
+
 ## 0.3.0 / 2022-01-18
 
 * Add config option :bothcase_name .

Gemfile

@@ -2,3 +2,7 @@ source "https://rubygems.org"
 
 # Specify your gem's dependencies in pg_ldap_sync.gemspec
 gemspec
+
+group :development do
+  gem "debug"
+end

README.md

@@ -20,16 +20,18 @@ It is meant to be started as a cron job.
 
 ## FEATURES:
 
+* User+group creation, deletion and changes in memberships are synchronized from LDAP to PostgreSQL
+* Nested groups/roles supported
 * Configurable per YAML config file
 * Can use Active Directory as LDAP-Server
-* Nested groups/roles supported
 * Set scope of considered users/groups on LDAP and PG side
 * Test mode which doesn't do any changes to the DBMS
 * Both LDAP and PG connections can be secured by SSL/TLS
+* NTLM and Kerberos authentication to LDAP server
 
 ## REQUIREMENTS:
 
-* Ruby-2.0+, JRuby-1.2+
+* Ruby-2.0+
 * LDAP-v3 server
 * PostgreSQL-server v9.0+
 
@@ -70,6 +72,11 @@ Run in modify-mode:
   pg_ldap_sync -c my_config.yaml -vv
 ```
 
+It is recommended to avoid granting permissions to synchronized users on the PostgreSQL server, but to grant permissions to groups instead.
+This is because `DROP USER` statements invoked when a user leaves otherwise fail due to depending objects.
+`DROP GROUP` equally fails if there are depending objects, but groups are typically more stable and removed rarely.
+
+
 ## TEST:
 There is a small test suite in the `test` directory that runs against an internal LDAP server and a PostgreSQL server. Ensure `pg_ctl`, `initdb` and `psql` commands are in the `PATH` like so:
 ```sh

Rakefile

@@ -2,6 +2,8 @@
 require "bundler/gem_tasks"
 require "rake/testtask"
 
+CLEAN.include "temp"
+
 Rake::TestTask.new(:test) do |t|
   t.libs << "test"
   t.libs << "lib"

config/sample-config.yaml

@@ -5,13 +5,26 @@
 # Connection parameters to LDAP server
 # see also: http://net-ldap.rubyforge.org/Net/LDAP.html#method-c-new
 ldap_connection:
-  host: localhost
+  host: ldapserver
   port: 389
   auth:
     method: :simple
     username: CN=username,OU=!Serviceaccounts,OU=company,DC=company,DC=de
     password: secret
 
+  # or GSSAPI / Kerberos authentication:
+  auth:
+    method: :gssapi
+    hostname: ldapserver.company.de
+    servicename: ldap # optional, defaults to "ldap"
+
+  # or GSS-SPNEGO / NTLM authentication
+  auth:
+    method: :gss_spnego
+    username: 'myuser'
+    password: 'secret'
+    domain: 'company.de' # optional
+
 # Search parameters for LDAP users which should be synchronized
 ldap_users:
   base: OU=company,OU=company,DC=company,DC=de
@@ -51,4 +64,5 @@ pg_groups:
   filter: NOT rolcanlogin AND NOT rolsuper
   # Options for CREATE RULE statements
   create_options: NOLOGIN
+  # Options for GRANT <role> TO <group> statements
   grant_options:

config/sample-config2.yaml

@@ -1,7 +1,12 @@
 # With this sample config the distinction between LDAP-synchronized
-# groups/users from is done by the membership to ldap_user and
-# ldap_group. These two roles have to be defined manally before
-# pg_ldap_sync can run.
+# groups/users from manually created PostgreSQL users is done by the
+# membership in ldap_user and ldap_group.
+# These two roles have to be defined manally before pg_ldap_sync can
+# run and all synchronized users/groups will become member of them
+# later on:
+#     CREATE GROUP ldap_groups;
+#     CREATE USER ldap_users;
+#
 
 # Connection parameters to LDAP server
 # see also: http://net-ldap.rubyforge.org/Net/LDAP.html#method-c-new
@@ -67,4 +72,5 @@ pg_groups:
   filter: oid IN (SELECT pam.member FROM pg_auth_members pam JOIN pg_roles pr ON pr.oid=pam.roleid WHERE pr.rolname='ldap_groups')
   # Options for CREATE RULE statements
   create_options: NOLOGIN IN ROLE ldap_groups
+  # Options for GRANT <role> TO <group> statements
   grant_options:

lib/pg_ldap_sync.rb

@@ -1,4 +1,5 @@
 require "pg_ldap_sync/application"
+require "pg_ldap_sync/compat"
 require "pg_ldap_sync/version"
 
 module PgLdapSync
@@ -8,7 +9,8 @@ module PgLdapSync
   class ApplicationExit < RuntimeError
     attr_reader :exitcode
 
-    def initialize(exitcode)
+    def initialize(exitcode, error=nil)
+      super(error)
       @exitcode = exitcode
     end
   end

lib/pg_ldap_sync/application.rb

@@ -52,13 +52,18 @@ class Application
 
   def search_ldap_users
     ldap_user_conf = @config[:ldap_users]
+    name_attribute = ldap_user_conf[:name_attribute]
 
     users = []
-    res = @ldap.search(:base => ldap_user_conf[:base], :filter => ldap_user_conf[:filter]) do |entry|
-      name = entry[ldap_user_conf[:name_attribute]].first
+    res = @ldap.search(
+          base: ldap_user_conf[:base],
+          filter: ldap_user_conf[:filter],
+          attributes: [name_attribute, :dn]
+    ) do |entry|
+      name = entry[name_attribute].first
 
       unless name
-        log.warn "user attribute #{ldap_user_conf[:name_attribute].inspect} not defined for #{entry.dn}"
+        log.warn "user attribute #{name_attribute.inspect} not defined for #{entry.dn}"
         next
       end
       log.info "found user-dn: #{entry.dn}"
@@ -87,17 +92,56 @@ class Application
     return users
   end
 
+  def retrieve_array_attribute(entry, attribute_name)
+    array = entry[attribute_name]
+    if array.empty?
+      # Possibly an attribute, which must be retrieved in several ranges
+
+      ranged_attr = entry.attribute_names.find { |n| n =~ /\A#{Regexp.escape(attribute_name)};range=/ }
+      if ranged_attr
+        entry_dn = entry.dn
+
+        loop do
+          array += entry[ranged_attr]
+          log.debug "retrieved attribute range #{ranged_attr.inspect} of dn #{entry_dn}"
+
+          if ranged_attr =~ /;range=\d\-\*\z/
+            break
+          end
+
+          attribute_with_range = ranged_attr.to_s.gsub(/;range=.*/, ";range=#{array.size}-*")
+          entry = @ldap.search(
+            base: entry_dn,
+            scope: Net::LDAP::SearchScope_BaseObject,
+            attributes: attribute_with_range).first
+
+          ranged_attr = entry.attribute_names.find { |n| n =~ /\A#{Regexp.escape(attribute_name)};range=/ }
+        end
+      end
+    else
+      # Values already received -> No ranged attribute
+    end
+    return array
+  end
+
   def search_ldap_groups
     ldap_group_conf = @config[:ldap_groups]
+    name_attribute = ldap_group_conf[:name_attribute]
+    member_attribute = ldap_group_conf[:member_attribute]
 
     groups = []
-    res = @ldap.search(:base => ldap_group_conf[:base], :filter => ldap_group_conf[:filter]) do |entry|
-      name = entry[ldap_group_conf[:name_attribute]].first
+    res = @ldap.search(
+          base: ldap_group_conf[:base],
+          filter: ldap_group_conf[:filter],
+          attributes: [name_attribute, member_attribute, :dn]
+    ) do |entry|
+      name = entry[name_attribute].first
 
       unless name
-        log.warn "user attribute #{ldap_group_conf[:name_attribute].inspect} not defined for #{entry.dn}"
+        log.warn "user attribute #{name_attribute.inspect} not defined for #{entry.dn}"
         next
       end
+
       log.info "found group-dn: #{entry.dn}"
 
       names = if ldap_group_conf[:bothcase_name]
@@ -111,7 +155,8 @@ class Application
       end
 
       names.each do |n|
-        groups << LdapRole.new(n, entry.dn, entry[ldap_group_conf[:member_attribute]])
+        group_members = retrieve_array_attribute(entry, member_attribute)
+        groups << LdapRole.new(n, entry.dn, group_members)
       end
       entry.each do |attribute, values|
         log.debug "   #{attribute}:"
@@ -127,7 +172,7 @@ class Application
   PgRole = Struct.new :name, :member_names
 
   # List of default roles taken from https://www.postgresql.org/docs/current/predefined-roles.html
-  PG_BUILTIN_ROLES = %w[ pg_read_all_data pg_write_all_data pg_read_all_settings pg_read_all_stats pg_stat_scan_tables pg_monitor pg_database_owner pg_signal_backend pg_read_server_files pg_write_server_files pg_execute_server_program ]
+  PG_BUILTIN_ROLES = %w[ pg_read_all_data pg_write_all_data pg_read_all_settings pg_read_all_stats pg_stat_scan_tables pg_monitor pg_database_owner pg_signal_backend pg_read_server_files pg_write_server_files pg_execute_server_program pg_checkpoint]
 
   def search_pg_users
     pg_users_conf = @config[:pg_users]
@@ -325,8 +370,26 @@ class Application
   def start!
     read_config_file(@config_fname)
 
+    ldap_conf = @config[:ldap_connection]
+    auth_meth = ldap_conf.dig(:auth, :method).to_s
+    if auth_meth == "gssapi"
+      begin
+        require 'net/ldap/auth_adapter/gssapi'
+      rescue LoadError => err
+        raise "#{err}\nTo use GSSAPI authentication please run:\n  gem install net-ldap-auth_adapter-gssapi"
+      end
+    elsif auth_meth == "gss_spnego"
+      begin
+        require 'net-ldap-gss-spnego'
+        # This doesn't work since this file is defined in net-ldap as a placeholder:
+        #   require 'net/ldap/auth_adapter/gss_spnego'
+      rescue LoadError => err
+        raise "#{err}\nTo use GSSAPI authentication please run:\n  gem install net-ldap-gss-spnego"
+      end
+    end
+
     # gather LDAP users and groups
-    @ldap = Net::LDAP.new @config[:ldap_connection]
+    @ldap = Net::LDAP.new ldap_conf
     ldap_users = uniq_names search_ldap_users
     ldap_groups = uniq_names search_ldap_groups
 
@@ -357,14 +420,14 @@ class Application
 
     # Determine exitcode
     if log.had_errors?
-      raise ErrorExit, 1
+      raise ErrorExit.new(1, log.first_error)
     end
   end
 
   def self.run(argv)
     s = self.new
     s.config_fname = '/etc/pg_ldap_sync.yaml'
-    s.log = Logger.new($stdout, @error_counters)
+    s.log = Logger.new($stdout)
     s.log.level = Logger::ERROR
 
     OptionParser.new do |opts|

lib/pg_ldap_sync/compat.rb

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+class Hash
+  # transform_keys was added in ruby-2.5
+  def transform_keys
+    map do |k, v|
+      [yield(k), v]
+    end.to_h
+  end unless method_defined? :transform_keys
+end

lib/pg_ldap_sync/logger.rb

@@ -2,23 +2,27 @@ require 'logger'
 
 module PgLdapSync
 class Logger < ::Logger
-  def initialize(io, counters)
+  def initialize(io)
     super(io)
     @counters = {}
   end
 
-  def add(severity, *args)
-    @counters[severity] ||= 0
-    @counters[severity] += 1
+  def add(severity, *args, &block)
     super
+    return unless [Logger::FATAL, Logger::ERROR].include?(severity)
+    @counters[severity] ||= block ? block.call : args.first
   end
 
   def had_logged?(severity)
-    @counters[severity] && @counters[severity] > 0
+    !!@counters[severity]
   end
 
   def had_errors?
     had_logged?(Logger::FATAL) || had_logged?(Logger::ERROR)
   end
+
+  def first_error
+    @counters[Logger::FATAL] || @counters[Logger::ERROR]
+  end
 end
 end

lib/pg_ldap_sync/version.rb

@@ -1,3 +1,3 @@
 module PgLdapSync
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

pg-ldap-sync.gemspec

@@ -19,12 +19,12 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
   spec.rdoc_options = %w[--main README.md --charset=UTF-8]
-  spec.required_ruby_version = ">= 2.4"
+  spec.required_ruby_version = ">= 2.3"
 
   spec.add_runtime_dependency "net-ldap", "~> 0.16"
   spec.add_runtime_dependency "kwalify", "~> 0.7"
   spec.add_runtime_dependency "pg", ">= 0.14", "< 2.0"
-  spec.add_development_dependency "ruby-ldapserver", "~> 0.3"
+  spec.add_development_dependency "ruby-ldapserver", "~> 0.7"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "bundler", ">= 1.16", "< 3.0"
   spec.add_development_dependency "rake", "~> 13.0"

test/fixtures/ldapdb.yaml

@@ -2,6 +2,9 @@
 dc=example,dc=com:
   cn:
   - Top object
+cn=Unknown Flintstone,dc=example,dc=com:
+  cn:
+  - Unknown Flintstone
 cn=Fred Flintstone,dc=example,dc=com:
   cn:
   - Fred Flintstone
@@ -36,3 +39,4 @@ cn=All Users,dc=example,dc=com:
   member:
   - cn=Wilmas,dc=example,dc=com
   - cn=Fred Flintstone,dc=example,dc=com
+  - cn=Unknown Flintstone,dc=example,dc=com

test/ldap_server.rb

@@ -17,11 +17,10 @@ class HashOperation < LDAP::Server::Operation
 
   def search(basedn, scope, deref, filter)
     basedn.downcase!
-
     case scope
     when LDAP::Server::BaseObject
       # client asked for single object by DN
-      obj = @hash[basedn]
+      obj = @hash.transform_keys(&:downcase)[basedn]
       raise LDAP::ResultError::NoSuchObject unless obj
       send_SearchResultEntry(basedn, obj) if LDAP::Server::Filter.run(filter, obj)
 

test/test_pg_ldap_sync.rb

@@ -41,7 +41,8 @@ class TestPgLdapSync < Minitest::Test
     # :ssl_cert_file    => "cert.pem",
     # :ssl_on_connect   => true,
       :operation_class  => HashOperation,
-      :operation_args   => @directory
+      :operation_args   => @directory,
+      :attribute_range_limit => 2
     )
     @ldap_server.run_tcpserver
   end
@@ -122,7 +123,7 @@ class TestPgLdapSync < Minitest::Test
   end
 
   def sync_with_config(config="config-ldapdb")
-    PgLdapSync::Application.run(["-c", "test/fixtures/#{config}.yaml"] + ($DEBUG ? ["-vv"] : ["--no-verbose"]))
+    PgLdapSync::Application.run(["-c", "test/fixtures/#{config}.yaml"] + ($DEBUG ? ["-vvv"] : ["--no-verbose"]))
   end
 
   def sync_to_fixture(fixture: "ldapdb", config: "config-ldapdb")