From d833228547ed9d5d3b79a8f17143512a9762f49d Mon Sep 17 00:00:00 2001 From: Jingwen Yang Date: Tue, 23 Aug 2022 11:33:17 +0800 Subject: [PATCH 01/25] Support synchronize groups with over 1500 users in Active Directory server --- config/sample-config2.yaml | 2 ++ config/schema.yaml | 3 +++ lib/pg_ldap_sync/application.rb | 40 ++++++++++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/config/sample-config2.yaml b/config/sample-config2.yaml index dd96ea1..fa29f12 100644 --- a/config/sample-config2.yaml +++ b/config/sample-config2.yaml @@ -38,6 +38,8 @@ ldap_groups: lowercase_name: false # this attribute must reference to all member DN's of the given group member_attribute: member + # must be true if group with over 1500 members for Active Directory server + need_member_range_retrieval: false # Connection parameters to PostgreSQL server # see also: http://rubydoc.info/gems/pg/PG/Connection#initialize-instance_method diff --git a/config/schema.yaml b/config/schema.yaml index 08abdb8..227de45 100644 --- a/config/schema.yaml +++ b/config/schema.yaml @@ -46,6 +46,9 @@ mapping: "member_attribute": type: str required: yes + "need_member_range_retrieval": + type: bool + required: no "pg_connection": type: any diff --git a/lib/pg_ldap_sync/application.rb b/lib/pg_ldap_sync/application.rb index d68c858..90c7ec2 100644 --- a/lib/pg_ldap_sync/application.rb +++ b/lib/pg_ldap_sync/application.rb @@ -85,6 +85,40 @@ class Application return users end + def load_group_members_by_range(group_dn) + range_start = 0 + all_group_members = [] + while true do + member_size = 0 + member_attribute_with_range = "#{@config[:ldap_groups][:member_attribute]};range=#{range_start}-*" + returned_member_attribute_with_range = "" + log.debug " current attribute for range retrieval ----> #{member_attribute_with_range} " + + res_entry = @ldap.search( + base: group_dn, + filter: Net::LDAP::Filter.eq('distinguishedName', group_dn), + attributes: member_attribute_with_range).first + + res_entry.each do |attribute, values| + next if "#{attribute}" == "dn" + returned_member_attribute_with_range = "#{attribute}" + log.debug " returned attribute --------> #{returned_member_attribute_with_range}" + member_size = values.count + values.each do |value| + log.debug " -----> #{value}" + all_group_members << value + end + break + end + + if returned_member_attribute_with_range == member_attribute_with_range + break + end + range_start = range_start + member_size + end + return all_group_members + end + def search_ldap_groups ldap_group_conf = @config[:ldap_groups] @@ -107,7 +141,11 @@ class Application end names.each do |n| - groups << LdapRole.new(n, entry.dn, entry[ldap_group_conf[:member_attribute]]) + group_members = entry[ldap_group_conf[:member_attribute]] + if group_members.count == 0 and ldap_group_conf[:need_member_range_retrieval] + group_members = load_group_members_by_range(entry.dn) + end + groups << LdapRole.new(n, entry.dn, group_members) end entry.each do |attribute, values| log.debug " #{attribute}:" From 917e716d384ee1bf088509a1a96a13c6d1c73982 Mon Sep 17 00:00:00 2001 From: Jingwen Yang Date: Wed, 24 Aug 2022 14:30:22 +0800 Subject: [PATCH 02/25] Instead of adding a new config option, use range retrieval when member_attribute includes ;range --- config/sample-config2.yaml | 5 +++-- config/schema.yaml | 3 --- lib/pg_ldap_sync/application.rb | 7 ++++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/config/sample-config2.yaml b/config/sample-config2.yaml index fa29f12..0145f3e 100644 --- a/config/sample-config2.yaml +++ b/config/sample-config2.yaml @@ -37,9 +37,10 @@ ldap_groups: # lowercase name for use as PG role name lowercase_name: false # this attribute must reference to all member DN's of the given group + # If LDAP server is Active Directory, it's better to append ";range" to member_attribue; + # otherwise, it can't synchronize groups with over 1500 users for AD server. + # Example for AD server: "member;range" member_attribute: member - # must be true if group with over 1500 members for Active Directory server - need_member_range_retrieval: false # Connection parameters to PostgreSQL server # see also: http://rubydoc.info/gems/pg/PG/Connection#initialize-instance_method diff --git a/config/schema.yaml b/config/schema.yaml index 227de45..08abdb8 100644 --- a/config/schema.yaml +++ b/config/schema.yaml @@ -46,9 +46,6 @@ mapping: "member_attribute": type: str required: yes - "need_member_range_retrieval": - type: bool - required: no "pg_connection": type: any diff --git a/lib/pg_ldap_sync/application.rb b/lib/pg_ldap_sync/application.rb index 90c7ec2..73d8728 100644 --- a/lib/pg_ldap_sync/application.rb +++ b/lib/pg_ldap_sync/application.rb @@ -90,7 +90,7 @@ class Application all_group_members = [] while true do member_size = 0 - member_attribute_with_range = "#{@config[:ldap_groups][:member_attribute]};range=#{range_start}-*" + member_attribute_with_range = "#{@config[:ldap_groups][:member_attribute]}=#{range_start}-*" returned_member_attribute_with_range = "" log.debug " current attribute for range retrieval ----> #{member_attribute_with_range} " @@ -141,8 +141,9 @@ class Application end names.each do |n| - group_members = entry[ldap_group_conf[:member_attribute]] - if group_members.count == 0 and ldap_group_conf[:need_member_range_retrieval] + member_attribute_sub_list = ldap_group_conf[:member_attribute].partition(";") + group_members = entry[member_attribute_sub_list[0]] + if group_members.count == 0 and member_attribute_sub_list[2] == "range" group_members = load_group_members_by_range(entry.dn) end groups << LdapRole.new(n, entry.dn, group_members) From ead4404c54daf76181a85eb4f208509e8581792f Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Wed, 30 Nov 2022 13:15:09 +0100 Subject: [PATCH 03/25] Add "temp" dir to "rake clean" --- Rakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Rakefile b/Rakefile index aa6e7af..11db99f 100644 --- a/Rakefile +++ b/Rakefile @@ -2,6 +2,8 @@ require "bundler/gem_tasks" require "rake/testtask" +CLEAN.include "temp" + Rake::TestTask.new(:test) do |t| t.libs << "test" t.libs << "lib" From f83b32f8d9647fba16b70b63bc1234ae6a715b8c Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Wed, 30 Nov 2022 13:24:27 +0100 Subject: [PATCH 04/25] Revert "Instead of adding a new config option, use range retrieval when member_attribute includes ;range" This reverts commit 917e716d384ee1bf088509a1a96a13c6d1c73982. --- config/sample-config2.yaml | 5 ++--- config/schema.yaml | 3 +++ lib/pg_ldap_sync/application.rb | 7 +++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/config/sample-config2.yaml b/config/sample-config2.yaml index 0145f3e..fa29f12 100644 --- a/config/sample-config2.yaml +++ b/config/sample-config2.yaml @@ -37,10 +37,9 @@ ldap_groups: # lowercase name for use as PG role name lowercase_name: false # this attribute must reference to all member DN's of the given group - # If LDAP server is Active Directory, it's better to append ";range" to member_attribue; - # otherwise, it can't synchronize groups with over 1500 users for AD server. - # Example for AD server: "member;range" member_attribute: member + # must be true if group with over 1500 members for Active Directory server + need_member_range_retrieval: false # Connection parameters to PostgreSQL server # see also: http://rubydoc.info/gems/pg/PG/Connection#initialize-instance_method diff --git a/config/schema.yaml b/config/schema.yaml index 08abdb8..227de45 100644 --- a/config/schema.yaml +++ b/config/schema.yaml @@ -46,6 +46,9 @@ mapping: "member_attribute": type: str required: yes + "need_member_range_retrieval": + type: bool + required: no "pg_connection": type: any diff --git a/lib/pg_ldap_sync/application.rb b/lib/pg_ldap_sync/application.rb index 73d8728..90c7ec2 100644 --- a/lib/pg_ldap_sync/application.rb +++ b/lib/pg_ldap_sync/application.rb @@ -90,7 +90,7 @@ class Application all_group_members = [] while true do member_size = 0 - member_attribute_with_range = "#{@config[:ldap_groups][:member_attribute]}=#{range_start}-*" + member_attribute_with_range = "#{@config[:ldap_groups][:member_attribute]};range=#{range_start}-*" returned_member_attribute_with_range = "" log.debug " current attribute for range retrieval ----> #{member_attribute_with_range} " @@ -141,9 +141,8 @@ class Application end names.each do |n| - member_attribute_sub_list = ldap_group_conf[:member_attribute].partition(";") - group_members = entry[member_attribute_sub_list[0]] - if group_members.count == 0 and member_attribute_sub_list[2] == "range" + group_members = entry[ldap_group_conf[:member_attribute]] + if group_members.count == 0 and ldap_group_conf[:need_member_range_retrieval] group_members = load_group_members_by_range(entry.dn) end groups << LdapRole.new(n, entry.dn, group_members) From f5b9587eb73e6c1045f934a063276930d20faa5e Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Wed, 30 Nov 2022 13:24:29 +0100 Subject: [PATCH 05/25] Revert "Support synchronize groups with over 1500 users in Active Directory server" This reverts commit d833228547ed9d5d3b79a8f17143512a9762f49d. --- config/sample-config2.yaml | 2 -- config/schema.yaml | 3 --- lib/pg_ldap_sync/application.rb | 40 +-------------------------------- 3 files changed, 1 insertion(+), 44 deletions(-) diff --git a/config/sample-config2.yaml b/config/sample-config2.yaml index fa29f12..dd96ea1 100644 --- a/config/sample-config2.yaml +++ b/config/sample-config2.yaml @@ -38,8 +38,6 @@ ldap_groups: lowercase_name: false # this attribute must reference to all member DN's of the given group member_attribute: member - # must be true if group with over 1500 members for Active Directory server - need_member_range_retrieval: false # Connection parameters to PostgreSQL server # see also: http://rubydoc.info/gems/pg/PG/Connection#initialize-instance_method diff --git a/config/schema.yaml b/config/schema.yaml index 227de45..08abdb8 100644 --- a/config/schema.yaml +++ b/config/schema.yaml @@ -46,9 +46,6 @@ mapping: "member_attribute": type: str required: yes - "need_member_range_retrieval": - type: bool - required: no "pg_connection": type: any diff --git a/lib/pg_ldap_sync/application.rb b/lib/pg_ldap_sync/application.rb index 90c7ec2..d68c858 100644 --- a/lib/pg_ldap_sync/application.rb +++ b/lib/pg_ldap_sync/application.rb @@ -85,40 +85,6 @@ class Application return users end - def load_group_members_by_range(group_dn) - range_start = 0 - all_group_members = [] - while true do - member_size = 0 - member_attribute_with_range = "#{@config[:ldap_groups][:member_attribute]};range=#{range_start}-*" - returned_member_attribute_with_range = "" - log.debug " current attribute for range retrieval ----> #{member_attribute_with_range} " - - res_entry = @ldap.search( - base: group_dn, - filter: Net::LDAP::Filter.eq('distinguishedName', group_dn), - attributes: member_attribute_with_range).first - - res_entry.each do |attribute, values| - next if "#{attribute}" == "dn" - returned_member_attribute_with_range = "#{attribute}" - log.debug " returned attribute --------> #{returned_member_attribute_with_range}" - member_size = values.count - values.each do |value| - log.debug " -----> #{value}" - all_group_members << value - end - break - end - - if returned_member_attribute_with_range == member_attribute_with_range - break - end - range_start = range_start + member_size - end - return all_group_members - end - def search_ldap_groups ldap_group_conf = @config[:ldap_groups] @@ -141,11 +107,7 @@ class Application end names.each do |n| - group_members = entry[ldap_group_conf[:member_attribute]] - if group_members.count == 0 and ldap_group_conf[:need_member_range_retrieval] - group_members = load_group_members_by_range(entry.dn) - end - groups << LdapRole.new(n, entry.dn, group_members) + groups << LdapRole.new(n, entry.dn, entry[ldap_group_conf[:member_attribute]]) end entry.each do |attribute, values| log.debug " #{attribute}:" From 81473fb2b45a612a935b192a2a0c0564dddf86d3 Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Thu, 1 Dec 2022 15:21:39 +0100 Subject: [PATCH 06/25] Re-add support for attribute ranges This is an alternative implementation of #37 . It doesn't need additional configuration, but enables range retrieval automatic. It also add some testing. Fixes #32 --- Gemfile | 5 ++++ lib/pg_ldap_sync/application.rb | 42 ++++++++++++++++++++++++++++++--- test/fixtures/ldapdb.yaml | 4 ++++ test/ldap_server.rb | 3 +-- test/test_pg_ldap_sync.rb | 5 ++-- 5 files changed, 52 insertions(+), 7 deletions(-) diff --git a/Gemfile b/Gemfile index dfa2a95..8ab954d 100644 --- a/Gemfile +++ b/Gemfile @@ -2,3 +2,8 @@ source "https://rubygems.org" # Specify your gem's dependencies in pg_ldap_sync.gemspec gemspec + +group :development do + gem "debug" + gem "ruby-ldapserver", git: "https://github.com/larskanis/ruby-ldapserver/" +end diff --git a/lib/pg_ldap_sync/application.rb b/lib/pg_ldap_sync/application.rb index d68c858..252140b 100644 --- a/lib/pg_ldap_sync/application.rb +++ b/lib/pg_ldap_sync/application.rb @@ -85,17 +85,52 @@ class Application return users end + def retrieve_array_attribute(entry, attribute_name) + array = entry[attribute_name] + if array.empty? + # Possibly an attribute, which must be retrieved in several ranges + + ranged_attr = entry.attribute_names.find { |n| n =~ /\A#{Regexp.escape(attribute_name)};range=/ } + if ranged_attr + entry_dn = entry.dn + + loop do + array += entry[ranged_attr] + log.debug "retrieved attribute range #{ranged_attr.inspect} of dn #{entry_dn}" + + if ranged_attr =~ /;range=\d\-\*\z/ + break + end + + attribute_with_range = ranged_attr.to_s.gsub(/;range=.*/, ";range=#{array.size}-*") + entry = @ldap.search( + base: entry_dn, + scope: Net::LDAP::SearchScope_BaseObject, + attributes: attribute_with_range).first + + ranged_attr = entry.attribute_names.find { |n| n =~ /\A#{Regexp.escape(attribute_name)};range=/ } + end + end + else + # Values already received -> No ranged attribute + end + return array + end + def search_ldap_groups ldap_group_conf = @config[:ldap_groups] + name_attribute = ldap_group_conf[:name_attribute] + member_attribute = ldap_group_conf[:member_attribute] groups = [] res = @ldap.search(:base => ldap_group_conf[:base], :filter => ldap_group_conf[:filter]) do |entry| - name = entry[ldap_group_conf[:name_attribute]].first + name = entry[name_attribute].first unless name - log.warn "user attribute #{ldap_group_conf[:name_attribute].inspect} not defined for #{entry.dn}" + log.warn "user attribute #{name_attribute.inspect} not defined for #{entry.dn}" next end + log.info "found group-dn: #{entry.dn}" names = if ldap_group_conf[:bothcase_name] @@ -107,7 +142,8 @@ class Application end names.each do |n| - groups << LdapRole.new(n, entry.dn, entry[ldap_group_conf[:member_attribute]]) + group_members = retrieve_array_attribute(entry, member_attribute) + groups << LdapRole.new(n, entry.dn, group_members) end entry.each do |attribute, values| log.debug " #{attribute}:" diff --git a/test/fixtures/ldapdb.yaml b/test/fixtures/ldapdb.yaml index cac7e90..08438c3 100644 --- a/test/fixtures/ldapdb.yaml +++ b/test/fixtures/ldapdb.yaml @@ -2,6 +2,9 @@ dc=example,dc=com: cn: - Top object +cn=Unknown Flintstone,dc=example,dc=com: + cn: + - Unknown Flintstone cn=Fred Flintstone,dc=example,dc=com: cn: - Fred Flintstone @@ -36,3 +39,4 @@ cn=All Users,dc=example,dc=com: member: - cn=Wilmas,dc=example,dc=com - cn=Fred Flintstone,dc=example,dc=com + - cn=Unknown Flintstone,dc=example,dc=com diff --git a/test/ldap_server.rb b/test/ldap_server.rb index 3bb1d4f..b89158e 100644 --- a/test/ldap_server.rb +++ b/test/ldap_server.rb @@ -17,11 +17,10 @@ class HashOperation < LDAP::Server::Operation def search(basedn, scope, deref, filter) basedn.downcase! - case scope when LDAP::Server::BaseObject # client asked for single object by DN - obj = @hash[basedn] + obj = @hash.transform_keys(&:downcase)[basedn] raise LDAP::ResultError::NoSuchObject unless obj send_SearchResultEntry(basedn, obj) if LDAP::Server::Filter.run(filter, obj) diff --git a/test/test_pg_ldap_sync.rb b/test/test_pg_ldap_sync.rb index 245e3bb..65a5aae 100644 --- a/test/test_pg_ldap_sync.rb +++ b/test/test_pg_ldap_sync.rb @@ -41,7 +41,8 @@ class TestPgLdapSync < Minitest::Test # :ssl_cert_file => "cert.pem", # :ssl_on_connect => true, :operation_class => HashOperation, - :operation_args => @directory + :operation_args => @directory, + :attribute_range_limit => 2 ) @ldap_server.run_tcpserver end @@ -122,7 +123,7 @@ class TestPgLdapSync < Minitest::Test end def sync_with_config(config="config-ldapdb") - PgLdapSync::Application.run(["-c", "test/fixtures/#{config}.yaml"] + ($DEBUG ? ["-vv"] : ["--no-verbose"])) + PgLdapSync::Application.run(["-c", "test/fixtures/#{config}.yaml"] + ($DEBUG ? ["-vvv"] : ["--no-verbose"])) end def sync_to_fixture(fixture: "ldapdb", config: "config-ldapdb") From 5a0782047236bcb2aec55e53c3247db4307d9600 Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Fri, 2 Dec 2022 08:05:42 +0100 Subject: [PATCH 07/25] Fix compat with ruby < 2.5 --- lib/pg_ldap_sync.rb | 1 + lib/pg_ldap_sync/compat.rb | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 lib/pg_ldap_sync/compat.rb diff --git a/lib/pg_ldap_sync.rb b/lib/pg_ldap_sync.rb index 21a6185..8d0a5e0 100644 --- a/lib/pg_ldap_sync.rb +++ b/lib/pg_ldap_sync.rb @@ -1,4 +1,5 @@ require "pg_ldap_sync/application" +require "pg_ldap_sync/compat" require "pg_ldap_sync/version" module PgLdapSync diff --git a/lib/pg_ldap_sync/compat.rb b/lib/pg_ldap_sync/compat.rb new file mode 100644 index 0000000..24f7845 --- /dev/null +++ b/lib/pg_ldap_sync/compat.rb @@ -0,0 +1,9 @@ +#!/usr/bin/env ruby + +class Hash + def transform_keys + map do |k, v| + [yield(k), v] + end.to_h + end unless method_defined? :transform_keys +end From f4d85e439e4b1a3944277818c754ff35b8464cb2 Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Fri, 2 Dec 2022 11:00:26 +0100 Subject: [PATCH 08/25] CI: Add Github actions --- .github/workflows/ci.yml | 81 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..20624e9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,81 @@ +name: CI + +on: [push, pull_request] + +jobs: + job_test_gem: + name: Test built gem + strategy: + fail-fast: false + matrix: + include: + - os: windows + ruby: "head" + PGVERSION: 15.1-1-windows-x64 + PGVER: "15" + - os: windows + ruby: "2.4" + PGVERSION: 9.4.26-1-windows-x64 + PGVER: "9.4" + - os: ubuntu + ruby: "head" + PGVER: "15" + - os: ubuntu + os_ver: "20.04" + ruby: "2.3" + PGVER: "9.3" + - os: macos + ruby: "head" + PGVERSION: 15.1-1-osx + PGVER: "15" + + runs-on: ${{ matrix.os }}-${{ matrix.os_ver || 'latest' }} + env: + PGVERSION: ${{ matrix.PGVERSION }} + PGVER: ${{ matrix.PGVER }} + + steps: + - uses: actions/checkout@v3 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + + - name: Download PostgreSQL Windows + if: matrix.os == 'windows' + run: | + Add-Type -AssemblyName System.IO.Compression.FileSystem + function Unzip { + param([string]$zipfile, [string]$outpath) + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath) + } + + $(new-object net.webclient).DownloadFile("http://get.enterprisedb.com/postgresql/postgresql-$env:PGVERSION-binaries.zip", "postgresql-binaries.zip") + Unzip "postgresql-binaries.zip" "." + echo "$pwd/pgsql/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + echo "PGUSER=$env:USERNAME" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + echo "PGPASSWORD=" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + md temp + icacls temp /grant "Everyone:(OI)(CI)F" /T + + - name: Download PostgreSQL Ubuntu + if: matrix.os == 'ubuntu' + run: | + echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main $PGVER" | sudo tee -a /etc/apt/sources.list.d/pgdg.list + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt-get -y update + sudo apt-get -y --allow-downgrades install postgresql-$PGVER libpq5=$PGVER* libpq-dev=$PGVER* + echo /usr/lib/postgresql/$PGVER/bin >> $GITHUB_PATH + + - name: Download PostgreSQL Macos + if: matrix.os == 'macos' + run: | + wget https://get.enterprisedb.com/postgresql/postgresql-$PGVERSION-binaries.zip && \ + sudo mkdir -p /Library/PostgreSQL && \ + sudo unzip postgresql-$PGVERSION-binaries.zip -d /Library/PostgreSQL/$PGVER && \ + echo /Library/PostgreSQL/$PGVER/bin >> $GITHUB_PATH + + - run: bundle install + + - name: Run specs + run: bundle exec rake test From bd4310a080a294918a9bacf8ac75e2417981c48c Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Fri, 2 Dec 2022 12:09:07 +0100 Subject: [PATCH 09/25] Downgrade required ruby version to 2.3 since we test it on Github Actions now --- pg-ldap-sync.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pg-ldap-sync.gemspec b/pg-ldap-sync.gemspec index 7072230..ae2691f 100644 --- a/pg-ldap-sync.gemspec +++ b/pg-ldap-sync.gemspec @@ -19,7 +19,7 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ["lib"] spec.rdoc_options = %w[--main README.md --charset=UTF-8] - spec.required_ruby_version = ">= 2.4" + spec.required_ruby_version = ">= 2.3" spec.add_runtime_dependency "net-ldap", "~> 0.16" spec.add_runtime_dependency "kwalify", "~> 0.7" From ee81807f494922e921e5217d78e4ca8e703f06c7 Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Fri, 2 Dec 2022 13:18:28 +0100 Subject: [PATCH 10/25] Add error text to exception when an error was logged --- lib/pg_ldap_sync.rb | 3 ++- lib/pg_ldap_sync/application.rb | 4 ++-- lib/pg_ldap_sync/logger.rb | 14 +++++++++----- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/pg_ldap_sync.rb b/lib/pg_ldap_sync.rb index 8d0a5e0..aa0cc86 100644 --- a/lib/pg_ldap_sync.rb +++ b/lib/pg_ldap_sync.rb @@ -9,7 +9,8 @@ module PgLdapSync class ApplicationExit < RuntimeError attr_reader :exitcode - def initialize(exitcode) + def initialize(exitcode, error=nil) + super(error) @exitcode = exitcode end end diff --git a/lib/pg_ldap_sync/application.rb b/lib/pg_ldap_sync/application.rb index 252140b..25a3e8f 100644 --- a/lib/pg_ldap_sync/application.rb +++ b/lib/pg_ldap_sync/application.rb @@ -384,14 +384,14 @@ class Application # Determine exitcode if log.had_errors? - raise ErrorExit, 1 + raise ErrorExit.new(1, log.first_error) end end def self.run(argv) s = self.new s.config_fname = '/etc/pg_ldap_sync.yaml' - s.log = Logger.new($stdout, @error_counters) + s.log = Logger.new($stdout) s.log.level = Logger::ERROR OptionParser.new do |opts| diff --git a/lib/pg_ldap_sync/logger.rb b/lib/pg_ldap_sync/logger.rb index 5fa43cd..8219cb8 100644 --- a/lib/pg_ldap_sync/logger.rb +++ b/lib/pg_ldap_sync/logger.rb @@ -2,23 +2,27 @@ require 'logger' module PgLdapSync class Logger < ::Logger - def initialize(io, counters) + def initialize(io) super(io) @counters = {} end - def add(severity, *args) - @counters[severity] ||= 0 - @counters[severity] += 1 + def add(severity, *args, &block) + return unless [Logger::FATAL, Logger::ERROR].include?(severity) + @counters[severity] ||= block ? block.call : args.first super end def had_logged?(severity) - @counters[severity] && @counters[severity] > 0 + !!@counters[severity] end def had_errors? had_logged?(Logger::FATAL) || had_logged?(Logger::ERROR) end + + def first_error + @counters[Logger::FATAL] || @counters[Logger::ERROR] + end end end From 40661830a49e280666945ae0ef7ece0a140ab5ae Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Fri, 2 Dec 2022 13:27:35 +0100 Subject: [PATCH 11/25] Exclude internal role of PostgreSQL-15 --- lib/pg_ldap_sync/application.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pg_ldap_sync/application.rb b/lib/pg_ldap_sync/application.rb index 25a3e8f..e589944 100644 --- a/lib/pg_ldap_sync/application.rb +++ b/lib/pg_ldap_sync/application.rb @@ -159,7 +159,7 @@ class Application PgRole = Struct.new :name, :member_names # List of default roles taken from https://www.postgresql.org/docs/current/predefined-roles.html - PG_BUILTIN_ROLES = %w[ pg_read_all_data pg_write_all_data pg_read_all_settings pg_read_all_stats pg_stat_scan_tables pg_monitor pg_database_owner pg_signal_backend pg_read_server_files pg_write_server_files pg_execute_server_program ] + PG_BUILTIN_ROLES = %w[ pg_read_all_data pg_write_all_data pg_read_all_settings pg_read_all_stats pg_stat_scan_tables pg_monitor pg_database_owner pg_signal_backend pg_read_server_files pg_write_server_files pg_execute_server_program pg_checkpoint] def search_pg_users pg_users_conf = @config[:pg_users] From ff6d6cce2213783955a9a33ab0f6cd5455ff3417 Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Fri, 2 Dec 2022 14:03:45 +0100 Subject: [PATCH 12/25] Fix logging with WARN, INFO, DEBUG level --- lib/pg_ldap_sync/logger.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pg_ldap_sync/logger.rb b/lib/pg_ldap_sync/logger.rb index 8219cb8..51358db 100644 --- a/lib/pg_ldap_sync/logger.rb +++ b/lib/pg_ldap_sync/logger.rb @@ -8,9 +8,9 @@ class Logger < ::Logger end def add(severity, *args, &block) + super return unless [Logger::FATAL, Logger::ERROR].include?(severity) @counters[severity] ||= block ? block.call : args.first - super end def had_logged?(severity) From eebbb159eaa285b4155813a3b62fd168fe927cac Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Fri, 2 Dec 2022 14:04:14 +0100 Subject: [PATCH 13/25] Add CREATE USER, GROUP statements as comments --- config/sample-config2.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/config/sample-config2.yaml b/config/sample-config2.yaml index dd96ea1..91bfae4 100644 --- a/config/sample-config2.yaml +++ b/config/sample-config2.yaml @@ -1,7 +1,10 @@ # With this sample config the distinction between LDAP-synchronized # groups/users from is done by the membership to ldap_user and # ldap_group. These two roles have to be defined manally before -# pg_ldap_sync can run. +# pg_ldap_sync can run: +# CREATE GROUP ldap_group; +# CREATE USER ldap_user; +# # Connection parameters to LDAP server # see also: http://net-ldap.rubyforge.org/Net/LDAP.html#method-c-new From f43d18d46a20c82e825cb978f5b6d8185fff353c Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Fri, 2 Dec 2022 14:04:43 +0100 Subject: [PATCH 14/25] Retrieve only necessary attributes from LDAP server --- lib/pg_ldap_sync/application.rb | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/lib/pg_ldap_sync/application.rb b/lib/pg_ldap_sync/application.rb index e589944..dcd6280 100644 --- a/lib/pg_ldap_sync/application.rb +++ b/lib/pg_ldap_sync/application.rb @@ -52,13 +52,18 @@ class Application def search_ldap_users ldap_user_conf = @config[:ldap_users] + name_attribute = ldap_user_conf[:name_attribute] users = [] - res = @ldap.search(:base => ldap_user_conf[:base], :filter => ldap_user_conf[:filter]) do |entry| - name = entry[ldap_user_conf[:name_attribute]].first + res = @ldap.search( + base: ldap_user_conf[:base], + filter: ldap_user_conf[:filter], + attributes: [name_attribute, :dn] + ) do |entry| + name = entry[name_attribute].first unless name - log.warn "user attribute #{ldap_user_conf[:name_attribute].inspect} not defined for #{entry.dn}" + log.warn "user attribute #{name_attribute.inspect} not defined for #{entry.dn}" next end log.info "found user-dn: #{entry.dn}" @@ -123,7 +128,11 @@ class Application member_attribute = ldap_group_conf[:member_attribute] groups = [] - res = @ldap.search(:base => ldap_group_conf[:base], :filter => ldap_group_conf[:filter]) do |entry| + res = @ldap.search( + base: ldap_group_conf[:base], + filter: ldap_group_conf[:filter], + attributes: [name_attribute, member_attribute, :dn] + ) do |entry| name = entry[name_attribute].first unless name From f1ee6e38cebd942d63c1374466aec24e6ccbaa1f Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Fri, 2 Dec 2022 16:33:53 +0100 Subject: [PATCH 15/25] Add CHANGELOG entry for VERSION 0.4.0 --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f76978..67d34c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## 0.4.0 / 2022-12-02 + +* Support groups with over 1500 users in Active Directory server. #32 +* Retrieve only necessary attributes from LDAP server. +* Add error text to exception, so that it's visible even if nothing is logged. +* Fix compatibility with PostgreSQL-15 +* Require ruby-2.3+ + + ## 0.3.0 / 2022-01-18 * Add config option :bothcase_name . From 7e5e24e92222dd236803fe6b0976c69e1d4cb980 Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Fri, 2 Dec 2022 16:47:58 +0100 Subject: [PATCH 16/25] Bump VERSION to 0.4.0 --- lib/pg_ldap_sync/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pg_ldap_sync/version.rb b/lib/pg_ldap_sync/version.rb index c2fba6c..0510f0c 100644 --- a/lib/pg_ldap_sync/version.rb +++ b/lib/pg_ldap_sync/version.rb @@ -1,3 +1,3 @@ module PgLdapSync - VERSION = "0.3.0" + VERSION = "0.4.0" end From 37b1bc17a07a0c520df7aeec066d1e7f8a7f14b1 Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Sat, 3 Dec 2022 12:23:26 +0100 Subject: [PATCH 17/25] Add ruby-version as a comment --- lib/pg_ldap_sync/compat.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pg_ldap_sync/compat.rb b/lib/pg_ldap_sync/compat.rb index 24f7845..8144a43 100644 --- a/lib/pg_ldap_sync/compat.rb +++ b/lib/pg_ldap_sync/compat.rb @@ -1,6 +1,7 @@ #!/usr/bin/env ruby class Hash + # transform_keys was added in ruby-2.5 def transform_keys map do |k, v| [yield(k), v] From 72505bf0619e94099c160276d3e589498e0a6a44 Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Tue, 6 Dec 2022 10:39:50 +0100 Subject: [PATCH 18/25] Switch back to ruby-ldapserver gem No need to use git any longer since ruby-ldapserver-0.7.0 --- Gemfile | 1 - pg-ldap-sync.gemspec | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 8ab954d..c5a7041 100644 --- a/Gemfile +++ b/Gemfile @@ -5,5 +5,4 @@ gemspec group :development do gem "debug" - gem "ruby-ldapserver", git: "https://github.com/larskanis/ruby-ldapserver/" end diff --git a/pg-ldap-sync.gemspec b/pg-ldap-sync.gemspec index ae2691f..48a9494 100644 --- a/pg-ldap-sync.gemspec +++ b/pg-ldap-sync.gemspec @@ -24,7 +24,7 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "net-ldap", "~> 0.16" spec.add_runtime_dependency "kwalify", "~> 0.7" spec.add_runtime_dependency "pg", ">= 0.14", "< 2.0" - spec.add_development_dependency "ruby-ldapserver", "~> 0.3" + spec.add_development_dependency "ruby-ldapserver", "~> 0.7" spec.add_development_dependency "minitest", "~> 5.0" spec.add_development_dependency "bundler", ">= 1.16", "< 3.0" spec.add_development_dependency "rake", "~> 13.0" From ccb3a1f889bf9392f9e474135d50388ce1840e3e Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Thu, 22 Dec 2022 18:30:18 +0100 Subject: [PATCH 19/25] Add comment about config option "grant_options" --- config/sample-config.yaml | 1 + config/sample-config2.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/config/sample-config.yaml b/config/sample-config.yaml index 0c610e4..2fdccb9 100644 --- a/config/sample-config.yaml +++ b/config/sample-config.yaml @@ -51,4 +51,5 @@ pg_groups: filter: NOT rolcanlogin AND NOT rolsuper # Options for CREATE RULE statements create_options: NOLOGIN + # Options for GRANT TO statements grant_options: diff --git a/config/sample-config2.yaml b/config/sample-config2.yaml index 91bfae4..83710e1 100644 --- a/config/sample-config2.yaml +++ b/config/sample-config2.yaml @@ -63,4 +63,5 @@ pg_groups: filter: oid IN (SELECT pam.member FROM pg_auth_members pam JOIN pg_roles pr ON pr.oid=pam.roleid WHERE pr.rolname='ldap_groups') # Options for CREATE RULE statements create_options: NOLOGIN IN ROLE ldap_groups + # Options for GRANT TO statements grant_options: From 20fd3118ed07b9ddcc401b927cc58690a7f10f25 Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Fri, 3 Feb 2023 19:42:18 +0100 Subject: [PATCH 20/25] Fix typo in names of roles --- config/sample-config2.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/sample-config2.yaml b/config/sample-config2.yaml index 83710e1..ef5b310 100644 --- a/config/sample-config2.yaml +++ b/config/sample-config2.yaml @@ -2,8 +2,8 @@ # groups/users from is done by the membership to ldap_user and # ldap_group. These two roles have to be defined manally before # pg_ldap_sync can run: -# CREATE GROUP ldap_group; -# CREATE USER ldap_user; +# CREATE GROUP ldap_groups; +# CREATE USER ldap_users; # # Connection parameters to LDAP server From 10d0f396943891cf1b29303b68d474c42126fd62 Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Fri, 3 Feb 2023 19:45:30 +0100 Subject: [PATCH 21/25] Add Kerberos and NTLM authentication support Fixes #41 --- config/sample-config.yaml | 14 +++++++++++++- lib/pg_ldap_sync/application.rb | 20 +++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/config/sample-config.yaml b/config/sample-config.yaml index 2fdccb9..d0273dd 100644 --- a/config/sample-config.yaml +++ b/config/sample-config.yaml @@ -5,13 +5,25 @@ # Connection parameters to LDAP server # see also: http://net-ldap.rubyforge.org/Net/LDAP.html#method-c-new ldap_connection: - host: localhost + host: ldapserver port: 389 auth: method: :simple username: CN=username,OU=!Serviceaccounts,OU=company,DC=company,DC=de password: secret + # or GSSAPI / Kerberos authentication: + auth: + method: :gssapi + hostname: ldapserver + + # or GSS-SPNEGO / NTLM authentication + auth: + method: :gss_spnego + domain: 'company.de' + username: 'myuser' + password: 'secret' + # Search parameters for LDAP users which should be synchronized ldap_users: base: OU=company,OU=company,DC=company,DC=de diff --git a/lib/pg_ldap_sync/application.rb b/lib/pg_ldap_sync/application.rb index dcd6280..2c6d9ae 100644 --- a/lib/pg_ldap_sync/application.rb +++ b/lib/pg_ldap_sync/application.rb @@ -361,8 +361,26 @@ class Application def start! read_config_file(@config_fname) + ldap_conf = @config[:ldap_connection] + auth_meth = ldap_conf.dig(:auth, :method).to_s + if auth_meth == "gssapi" + begin + require 'net/ldap/auth_adapter/gssapi' + rescue LoadError => err + raise "#{err}\nTo use GSSAPI authentication please run:\n gem install net-ldap-auth_adapter-gssapi" + end + elsif auth_meth == "gss_spnego" + begin + require 'net-ldap-gss-spnego' + # This doesn't work since this file is defined in net-ldap as a placeholder: + # require 'net/ldap/auth_adapter/gss_spnego' + rescue LoadError => err + raise "#{err}\nTo use GSSAPI authentication please run:\n gem install net-ldap-gss-spnego" + end + end + # gather LDAP users and groups - @ldap = Net::LDAP.new @config[:ldap_connection] + @ldap = Net::LDAP.new ldap_conf ldap_users = uniq_names search_ldap_users ldap_groups = uniq_names search_ldap_groups From a3b421f945064884944b830c97452baf4399c3e1 Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Fri, 3 Feb 2023 19:54:51 +0100 Subject: [PATCH 22/25] Add NTLM and Kerberos to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4169b93..0a96955 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ It is meant to be started as a cron job. * Set scope of considered users/groups on LDAP and PG side * Test mode which doesn't do any changes to the DBMS * Both LDAP and PG connections can be secured by SSL/TLS +* NTLM and Kerberos authentication to LDAP server ## REQUIREMENTS: From 8b06d6b4af3958a2ab83d48da908900f5b28f3ad Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Fri, 3 Feb 2023 19:55:21 +0100 Subject: [PATCH 23/25] Prepare release 0.5.0 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67d34c7..159dc60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.5.0 / 2023-02-03 + +* Add Kerberos and NTLM authentication support + + ## 0.4.0 / 2022-12-02 * Support groups with over 1500 users in Active Directory server. #32 From 0166bf8cdbbbc84b04dc82535d9a01b292c1b2ab Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Sat, 4 Feb 2023 18:56:34 +0100 Subject: [PATCH 24/25] Add optional keys to sample config and use FQDN for GSSAPI --- config/sample-config.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/sample-config.yaml b/config/sample-config.yaml index d0273dd..f901733 100644 --- a/config/sample-config.yaml +++ b/config/sample-config.yaml @@ -15,14 +15,15 @@ ldap_connection: # or GSSAPI / Kerberos authentication: auth: method: :gssapi - hostname: ldapserver + hostname: ldapserver.company.de + servicename: ldap # optional, defaults to "ldap" # or GSS-SPNEGO / NTLM authentication auth: method: :gss_spnego - domain: 'company.de' username: 'myuser' password: 'secret' + domain: 'company.de' # optional # Search parameters for LDAP users which should be synchronized ldap_users: From 5b994514c9f4d0ae1abe602e58becf3e2ad05f5e Mon Sep 17 00:00:00 2001 From: Lars Kanis Date: Sat, 1 Apr 2023 15:09:45 +0200 Subject: [PATCH 25/25] Improve documentation in README and config file --- README.md | 10 ++++++++-- config/sample-config2.yaml | 8 +++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0a96955..a522060 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,10 @@ It is meant to be started as a cron job. ## FEATURES: +* User+group creation, deletion and changes in memberships are synchronized from LDAP to PostgreSQL +* Nested groups/roles supported * Configurable per YAML config file * Can use Active Directory as LDAP-Server -* Nested groups/roles supported * Set scope of considered users/groups on LDAP and PG side * Test mode which doesn't do any changes to the DBMS * Both LDAP and PG connections can be secured by SSL/TLS @@ -30,7 +31,7 @@ It is meant to be started as a cron job. ## REQUIREMENTS: -* Ruby-2.0+, JRuby-1.2+ +* Ruby-2.0+ * LDAP-v3 server * PostgreSQL-server v9.0+ @@ -71,6 +72,11 @@ Run in modify-mode: pg_ldap_sync -c my_config.yaml -vv ``` +It is recommended to avoid granting permissions to synchronized users on the PostgreSQL server, but to grant permissions to groups instead. +This is because `DROP USER` statements invoked when a user leaves otherwise fail due to depending objects. +`DROP GROUP` equally fails if there are depending objects, but groups are typically more stable and removed rarely. + + ## TEST: There is a small test suite in the `test` directory that runs against an internal LDAP server and a PostgreSQL server. Ensure `pg_ctl`, `initdb` and `psql` commands are in the `PATH` like so: ```sh diff --git a/config/sample-config2.yaml b/config/sample-config2.yaml index ef5b310..d18acab 100644 --- a/config/sample-config2.yaml +++ b/config/sample-config2.yaml @@ -1,7 +1,9 @@ # With this sample config the distinction between LDAP-synchronized -# groups/users from is done by the membership to ldap_user and -# ldap_group. These two roles have to be defined manally before -# pg_ldap_sync can run: +# groups/users from manually created PostgreSQL users is done by the +# membership in ldap_user and ldap_group. +# These two roles have to be defined manally before pg_ldap_sync can +# run and all synchronized users/groups will become member of them +# later on: # CREATE GROUP ldap_groups; # CREATE USER ldap_users; #