From d833228547ed9d5d3b79a8f17143512a9762f49d Mon Sep 17 00:00:00 2001 From: Jingwen Yang Date: Tue, 23 Aug 2022 11:33:17 +0800 Subject: [PATCH 1/2] Support synchronize groups with over 1500 users in Active Directory server --- config/sample-config2.yaml | 2 ++ config/schema.yaml | 3 +++ lib/pg_ldap_sync/application.rb | 40 ++++++++++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/config/sample-config2.yaml b/config/sample-config2.yaml index dd96ea1..fa29f12 100644 --- a/config/sample-config2.yaml +++ b/config/sample-config2.yaml @@ -38,6 +38,8 @@ ldap_groups: lowercase_name: false # this attribute must reference to all member DN's of the given group member_attribute: member + # must be true if group with over 1500 members for Active Directory server + need_member_range_retrieval: false # Connection parameters to PostgreSQL server # see also: http://rubydoc.info/gems/pg/PG/Connection#initialize-instance_method diff --git a/config/schema.yaml b/config/schema.yaml index 08abdb8..227de45 100644 --- a/config/schema.yaml +++ b/config/schema.yaml @@ -46,6 +46,9 @@ mapping: "member_attribute": type: str required: yes + "need_member_range_retrieval": + type: bool + required: no "pg_connection": type: any diff --git a/lib/pg_ldap_sync/application.rb b/lib/pg_ldap_sync/application.rb index d68c858..90c7ec2 100644 --- a/lib/pg_ldap_sync/application.rb +++ b/lib/pg_ldap_sync/application.rb @@ -85,6 +85,40 @@ class Application return users end + def load_group_members_by_range(group_dn) + range_start = 0 + all_group_members = [] + while true do + member_size = 0 + member_attribute_with_range = "#{@config[:ldap_groups][:member_attribute]};range=#{range_start}-*" + returned_member_attribute_with_range = "" + log.debug " current attribute for range retrieval ----> #{member_attribute_with_range} " + + res_entry = @ldap.search( + base: group_dn, + filter: Net::LDAP::Filter.eq('distinguishedName', group_dn), + attributes: member_attribute_with_range).first + + res_entry.each do |attribute, values| + next if "#{attribute}" == "dn" + returned_member_attribute_with_range = "#{attribute}" + log.debug " returned attribute --------> #{returned_member_attribute_with_range}" + member_size = values.count + values.each do |value| + log.debug " -----> #{value}" + all_group_members << value + end + break + end + + if returned_member_attribute_with_range == member_attribute_with_range + break + end + range_start = range_start + member_size + end + return all_group_members + end + def search_ldap_groups ldap_group_conf = @config[:ldap_groups] @@ -107,7 +141,11 @@ class Application end names.each do |n| - groups << LdapRole.new(n, entry.dn, entry[ldap_group_conf[:member_attribute]]) + group_members = entry[ldap_group_conf[:member_attribute]] + if group_members.count == 0 and ldap_group_conf[:need_member_range_retrieval] + group_members = load_group_members_by_range(entry.dn) + end + groups << LdapRole.new(n, entry.dn, group_members) end entry.each do |attribute, values| log.debug " #{attribute}:" From 917e716d384ee1bf088509a1a96a13c6d1c73982 Mon Sep 17 00:00:00 2001 From: Jingwen Yang Date: Wed, 24 Aug 2022 14:30:22 +0800 Subject: [PATCH 2/2] Instead of adding a new config option, use range retrieval when member_attribute includes ;range --- config/sample-config2.yaml | 5 +++-- config/schema.yaml | 3 --- lib/pg_ldap_sync/application.rb | 7 ++++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/config/sample-config2.yaml b/config/sample-config2.yaml index fa29f12..0145f3e 100644 --- a/config/sample-config2.yaml +++ b/config/sample-config2.yaml @@ -37,9 +37,10 @@ ldap_groups: # lowercase name for use as PG role name lowercase_name: false # this attribute must reference to all member DN's of the given group + # If LDAP server is Active Directory, it's better to append ";range" to member_attribue; + # otherwise, it can't synchronize groups with over 1500 users for AD server. + # Example for AD server: "member;range" member_attribute: member - # must be true if group with over 1500 members for Active Directory server - need_member_range_retrieval: false # Connection parameters to PostgreSQL server # see also: http://rubydoc.info/gems/pg/PG/Connection#initialize-instance_method diff --git a/config/schema.yaml b/config/schema.yaml index 227de45..08abdb8 100644 --- a/config/schema.yaml +++ b/config/schema.yaml @@ -46,9 +46,6 @@ mapping: "member_attribute": type: str required: yes - "need_member_range_retrieval": - type: bool - required: no "pg_connection": type: any diff --git a/lib/pg_ldap_sync/application.rb b/lib/pg_ldap_sync/application.rb index 90c7ec2..73d8728 100644 --- a/lib/pg_ldap_sync/application.rb +++ b/lib/pg_ldap_sync/application.rb @@ -90,7 +90,7 @@ class Application all_group_members = [] while true do member_size = 0 - member_attribute_with_range = "#{@config[:ldap_groups][:member_attribute]};range=#{range_start}-*" + member_attribute_with_range = "#{@config[:ldap_groups][:member_attribute]}=#{range_start}-*" returned_member_attribute_with_range = "" log.debug " current attribute for range retrieval ----> #{member_attribute_with_range} " @@ -141,8 +141,9 @@ class Application end names.each do |n| - group_members = entry[ldap_group_conf[:member_attribute]] - if group_members.count == 0 and ldap_group_conf[:need_member_range_retrieval] + member_attribute_sub_list = ldap_group_conf[:member_attribute].partition(";") + group_members = entry[member_attribute_sub_list[0]] + if group_members.count == 0 and member_attribute_sub_list[2] == "range" group_members = load_group_members_by_range(entry.dn) end groups << LdapRole.new(n, entry.dn, group_members)