#!/usr/bin/env ruby
require File.dirname(__FILE__) + '/../config/environment.rb'
require 'pixways'
require 'optparse'
require 'optparse/time'
include Pixways
options = {}
options[:last_run] = (Time.now-7.days).strftime('%Y-%m-%d')
options[:field] = 'file_name'
options[:provider] = ''
OptionParser.new do |opts|
opts.banner = "Usage: duplicates_pp2 [options]"
opts.on("-p provider_string_key", "--provider provider_string_key", String, "Duplicates will be processed for this provider only") do |l|
options[:provider] = l
end
opts.on("-f field_name", "--field field_name", String, "Field used to compare the pictures, is 'file_name' by default") do |l|
options[:field] = l
end
opts.on("-l date", "--last_run date", String, "Date to retrieve lists pictures from, Format : YYYY-MM-DD, is 7 days before today by default. Put empty string (' ') to get all pictures.") do |l|
options[:last_run] = l
end
opts.on("-N", "--no-delete", "Duplicates won't de deleted, just listed") do |l|
options[:no_delete] = true
end
opts.on_tail("-h", "--help", "Show this message") do
puts opts
exit
end
end.parse!
pp2_list = []
list_folder = "/tmp/duplicates_pp2/#{Time.now.strftime('%Y%m%d')}/"
log_filename = "duplicates_pp2_#{Time.now.strftime('%Y-%m-%d-%Hh%Mm%S')}.log"
dup_list_filename = "duplicates_pp2_#{Time.now.strftime('%Y-%m-%d-%Hh%Mm%S')}.txt"
mail_body_filename = "duplicates_pp2_#{Time.now.strftime('%Y-%m-%d-%Hh%Mm%S')}_mail.html"
FileUtils.mkdir_p(list_folder) unless File.directory?(list_folder)
file_log = File.open(list_folder+log_filename,"a+")
file_mail_body = File.open(list_folder+mail_body_filename,"a+")
file_mail_body.write("Duplicates PA/PP2 from #{options[:last_run]} to #{Time.now().strftime('%Y-%m-%d')}
")
nb_error = 0
total_dup = 0
duplicates_lists = ""
STDOUT.sync = true
file_log.write("***** DEBUT - #{Time.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
begin
config_db_pp2 = Rails.configuration.database_configuration["pixpalace2"]
pp2_client = Mysql2::Client.new(:host => config_db_pp2["host"],:database => config_db_pp2["database"],:username => config_db_pp2["username"], :password => config_db_pp2["password"])
rescue Mysql2::Error => e
file_log.write("[#{Time.now().strftime('%Y-%m-%d %H:%M:%S')}] [ERROR] #{e.message} on #{config_db_pp2['host']}\n")
file_log.close
nb_error += 1
return
end
if options[:provider].blank?
#retrieve list of providers with pictures on PP2
begin
pp2_prov=pp2_client.query("select p.string_key from providers p join images i on i.provider_id=p.id group by i.provider_id;")
pp2_client.close
rescue Mysql2::Error => e
file_log.write("[#{Time.now().strftime('%Y-%m-%d %H:%M:%S')}] [ERROR] #{e.message} on #{config_db_pp2['host']} while retrieving list of providers with pictures on PP2\n")
file_log.close
pp2_client.close
nb_error += 1
return
end
else
pp2_prov = [{"string_key"=>options[:provider].to_s}]
end
file_log.write("___________________________________\n")
pp2_prov.each do |q|
q.each do |sk,p|
pp2_list_file = list_folder+p.to_s+"_pp2.txt"
pa_list_file = list_folder+p.to_s+"_pa.txt"
pp2_list_sorted_file = list_folder+p.to_s+"_sorted_pp2.txt"
pa_list_sorted_file = list_folder+p.to_s+"_sorted_pa.txt"
duplicates_list_file = list_folder+Time.now().strftime("%Y-%m-%d")+"_"+p.to_s+"_duplicates.txt"
file_log.write("*** #{p.to_s} ***\n")
#retrieve list of PP2 pictures of the provider
file_log.write("[#{Time.now().strftime('%Y-%m-%d %H:%M:%S')}] Retrieve list of PP2 pictures of #{p.to_s} created from #{options[:last_run]}\n")
begin
config_db_pp2 = Rails.configuration.database_configuration["pixpalace2"]
pp2_client = Mysql2::Client.new(:host => config_db_pp2["host"],:database => config_db_pp2["database"],:username => config_db_pp2["username"], :password => config_db_pp2["password"])
rescue Mysql2::Error => e
file_log.write("[#{Time.now().strftime('%Y-%m-%d %H:%M:%S')}] [ERROR] #{e.message} on #{config_db_pp2["host"]}\n")
file_log.close
nb_error += 1
return
end
begin
pp2_pics_by_prov=pp2_client.query("select #{options[:field]} from images where provider_id=(select id from providers where string_key='#{p.to_s}') and content_error=0 and (created_at >= '#{options[:last_run]}' or created_at is NULL);")
pp2_list = File.open(pp2_list_file,"a+")
file_log.write("Nb PP2 pictures : #{pp2_pics_by_prov.count}\n")
pp2_pics_by_prov.each do |i|
pp2_list.write(i["#{options[:field]}"].to_s+"\n")
end
pp2_list.close
pp2_client.close
rescue Mysql2::Error => e
file_log.write("[#{Time.now().strftime('%Y-%m-%d %H:%M:%S')}] [ERROR] #{e.message} on #{config_db_pp2['host']}\n")
pp2_list.close
pp2_client.close
nb_error += 1
break
end
#retrieve list of PA pictures of the provider
#process is longer with ActiveRecord
=begin
file_log.write("Retrieve list of PA pictures of #{p.to_s} - #{Time.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
pa_pics_by_prov = Image.joins(:provider).where(providers: {string_key: p.to_s}, content_error: 0).select(:file_name)
pa_list = File.open(pa_list_file,"a+")
file_log.write("Nb PA pictures : #{pa_pics_by_prov.count}\n")
pa_pics_by_prov.each do |i|
pa_list.write(i.file_name+"\n")
end
pa_list.close
=end
file_log.write("[#{Time.now().strftime('%Y-%m-%d %H:%M:%S')}] Retrieve list of PA pictures of #{p.to_s} created from #{options[:last_run]}\n")
begin
config_db_pa = Rails.configuration.database_configuration["production"]
pa_client = Mysql2::Client.new(:host => "localhost",:database => config_db_pa["database"],:username => config_db_pa["username"], :password => config_db_pa["password"])
rescue Mysql2::Error => e
file_log.write("[#{Time.now().strftime('%Y-%m-%d %H:%M:%S')}] [ERROR] #{e.message}\n")
file_log.close
nb_error += 1
return
end
begin
pa_pics_by_prov=pa_client.query("select #{options[:field]}, hires_location from images where provider_id=(select id from providers where string_key='#{p.to_s}') and content_error=0 and private_image=0 and (created_at >= '#{options[:last_run]}' or created_at is NULL);")
pa_list = File.open(pa_list_file,"a+")
file_log.write("Nb PA pictures : #{pa_pics_by_prov.count}\n")
pa_pics_by_prov.each do |i|
pa_list.write(i["#{options[:field]}"].to_s+"\n") unless (!i["hires_location"].blank? && i["hires_location"].include?('/usr/PPserver/PA_only'))
end
pa_list.close
pa_client.close
rescue Mysql2::Error => e
file_log.write("[#{Time.now().strftime('%Y-%m-%d %H:%M:%S')}] [ERROR] #{e.message}\n")
pa_client.close
nb_error += 1
break
end
#sort and compare list to create duplicates list
system("sort #{pp2_list_file} > #{pp2_list_sorted_file} && rm -f #{pp2_list_file}")
system("sort #{pa_list_file} > #{pa_list_sorted_file} && rm -f #{pa_list_file}")
system("comm -12 #{pp2_list_sorted_file} #{pa_list_sorted_file} > #{duplicates_list_file} && rm -f #{pp2_list_sorted_file}; rm -f #{pa_list_sorted_file}")
if File.zero?(duplicates_list_file)
file_log.write("[#{Time.now().strftime('%Y-%m-%d %H:%M:%S')}] No duplicate for #{p.to_s}\n")
File.delete(duplicates_list_file)
else
nb_dup = `wc -l #{duplicates_list_file}`.to_i
total_dup += nb_dup
file_log.write("[#{Time.now().strftime('%Y-%m-%d %H:%M:%S')}] #{nb_dup} duplicates for #{p.to_s} listed in #{duplicates_list_file}\n")
file_mail_body.write("-#{nb_dup} duplicates for #{p.to_s}
")
duplicates_lists += " #{duplicates_list_file}"
unless options[:no_delete]
file_log.write("[#{Time.now().strftime('%Y-%m-%d %H:%M:%S')}] Delete #{nb_dup} duplicates for #{p.to_s}\n")
imgs_list = []
File.open(duplicates_list_file,"rt").each do |i_field|
imgs_list << i_field.strip
end
image_deletion(imgs_list,'duplicates_pa_pp2','pp2',options[:field],options[:provider])
end
end
file_log.write("___________________________________\n")
end
end
file_log.write("\n***** FIN #{Time.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
file_log.close
file_mail_body.write("Total : #{total_dup} duplicates deleted on PP2
")
if nb_error > 0
file_mail_body.write("#{nb_error} errors during process, check log file attached
")
else
file_mail_body.write('No error during process.')
end
File.delete(list_folder+log_filename) if File.zero?(list_folder+log_filename)
File.delete(list_folder+dup_list_filename) if File.zero?(list_folder+dup_list_filename)
file_mail_body.write('')
file_mail_body.close
if system("sendemail -f support@pixpalace.com -t support@pixpalace.com info@pixpalace.com -s #{ENV['SMTP_ADDRESS']} -u 'Duplicates PA/PP2' -o message-file=#{list_folder+mail_body_filename} message-charset=utf-8 -a #{list_folder+log_filename} #{duplicates_lists} -l /var/log/pixways/duplicates_pp2_sendemail.log")
File.delete(list_folder+mail_body_filename)
end